From f4c3a327ca0030fed4326a004e717bb62e0e969e Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Thu, 5 Nov 2015 19:08:14 -0800 Subject: [PATCH 01/94] Add more detail to README's authentication section --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b9867fd198f..4b07bc7579d1 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,19 @@ There are multiple ways to authenticate to use Google Cloud services. 1. When using `gcloud-java` libraries from within Compute/App Engine, no additional authentication steps are necessary. 2. When using `gcloud-java` libraries elsewhere, there are two options: - * [Generate a JSON service account key](https://cloud.google.com/storage/docs/authentication?hl=en#service_accounts). Supply a path to the downloaded JSON credentials file when building the options supplied to datastore/storage constructor. + * [Generate a JSON service account key](https://cloud.google.com/storage/docs/authentication?hl=en#service_accounts). After downloading that key, you must do one of the following: + * Define the environment variable GOOGLE_APPLICATION_CREDENTIALS to be the location of the key. For example, `export GOOGLE_APPLICATION_CREDENTIALS=/path/to/my/key.json` + * Supply the downloaded JSON credentials file when building the options supplied to datastore/storage constructor. For example, `StorageOptions.builder().authCredentials(AuthCredentials.createForJson(new FileInputStream("/path/to/my/key.json")).build().service()` returns a `Storage` object that has the necessary permissions. * If running locally for development/testing, you can use use [Google Cloud SDK](https://cloud.google.com/sdk/?hl=en). To use the SDK authentication, [download the SDK](https://cloud.google.com/sdk/?hl=en) if you haven't already. Then login using the SDK (`gcloud auth login` in command line), and set your current project using `gcloud config set project PROJECT_ID`. +`gcloud-java` looks for credentials in the following order, stopping once it finds credentials: + +1. Credentials supplied to the `DatastoreOptions`/`ServiceOptions` builder +2. App Engine credentials +3. Key file pointed to by the GOOGLE_APPLICATION_CREDENTIALS environment variable +4. Google SDK credentials +5. Compute Engine credentials + Google Cloud Datastore ---------------------- From fe57e9865b20a4dee394c8042cd9306a20291149 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Thu, 5 Nov 2015 19:57:55 -0800 Subject: [PATCH 02/94] Add project ID order and clean docs language --- README.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4b07bc7579d1..d6da19e83ed0 100644 --- a/README.md +++ b/README.md @@ -46,17 +46,30 @@ There are multiple ways to authenticate to use Google Cloud services. 2. When using `gcloud-java` libraries elsewhere, there are two options: * [Generate a JSON service account key](https://cloud.google.com/storage/docs/authentication?hl=en#service_accounts). After downloading that key, you must do one of the following: * Define the environment variable GOOGLE_APPLICATION_CREDENTIALS to be the location of the key. For example, `export GOOGLE_APPLICATION_CREDENTIALS=/path/to/my/key.json` - * Supply the downloaded JSON credentials file when building the options supplied to datastore/storage constructor. For example, `StorageOptions.builder().authCredentials(AuthCredentials.createForJson(new FileInputStream("/path/to/my/key.json")).build().service()` returns a `Storage` object that has the necessary permissions. + * Supply the JSON credentials file when building the service options. For example, this Storage object has the necessary permissions to interact with your Google Cloud Storage data: + ```java + Storage storage = StorageOptions.builder() + .authCredentials(AuthCredentials.createForJson(new FileInputStream("/path/to/my/key.json")) + .build() + .service(); + ``` * If running locally for development/testing, you can use use [Google Cloud SDK](https://cloud.google.com/sdk/?hl=en). To use the SDK authentication, [download the SDK](https://cloud.google.com/sdk/?hl=en) if you haven't already. Then login using the SDK (`gcloud auth login` in command line), and set your current project using `gcloud config set project PROJECT_ID`. `gcloud-java` looks for credentials in the following order, stopping once it finds credentials: -1. Credentials supplied to the `DatastoreOptions`/`ServiceOptions` builder +1. Credentials supplied when building the service options 2. App Engine credentials 3. Key file pointed to by the GOOGLE_APPLICATION_CREDENTIALS environment variable -4. Google SDK credentials +4. Google Cloud SDK credentials 5. Compute Engine credentials +Note that this sequence is different than the order in which `gcloud-java` determines the project ID. The project ID is determined in the following order: + +1. Project ID supplied when building the service options +2. Project ID specified by the environment variable `GCLOUD_PROJECT` +3. App Engine project ID +4. Google Cloud SDK project ID + Google Cloud Datastore ---------------------- From 7a723996f7c352fd93ceae91f41e474de256873d Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 6 Nov 2015 11:31:14 +0100 Subject: [PATCH 03/94] Add iterateAll method to Page, update example and snippet --- .../src/main/java/com/google/gcloud/Page.java | 22 ++++++++++-- .../main/java/com/google/gcloud/PageImpl.java | 35 +++++++++++++++++++ .../java/com/google/gcloud/PageImplTest.java | 31 ++++++++++++---- .../gcloud/examples/StorageExample.java | 19 ++++------ .../gcloud/storage/RemoteGcsHelperTest.java | 6 ++++ 5 files changed, 91 insertions(+), 22 deletions(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/Page.java b/gcloud-java-core/src/main/java/com/google/gcloud/Page.java index 1b7754562716..2819b56a17a0 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/Page.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/Page.java @@ -16,11 +16,22 @@ package com.google.gcloud; +import java.util.Iterator; + /** * Interface for Google Cloud paginated results. * *

- * A typical {@code Page} usage: + * Use {@code Page} to iterate through all values (also in next pages): + * 

 {@code
+ * Page page = ...; // get a Page instance
+ * Iterator iterator = page.iterateAll();
+ * while (iterator.hasNext()) {
+ *   T value = iterator.next();
+ *   // do something with value
+ * }}
+ *

+ * Or handle pagination explicitly: * 

 {@code
  * Page page = ...; // get a Page instance
  * while (page != null) {
@@ -28,8 +39,7 @@
  *     // do something with value
  *   }
  *   page = page.nextPage();
- * }
- * }
+ * }} */ public interface Page { @@ -38,6 +48,12 @@ public interface Page { */ Iterable values(); + /** + * Returns an iterator for all values, possibly also in the next pages. Once current page's values + * are traversed the iterator fetches next page, if any. + */ + Iterator iterateAll(); + /** * Returns the cursor for the nextPage or {@code null} if no more results. */ diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java b/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java index 3925079c8d4b..5f70b22ec577 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java @@ -16,8 +16,11 @@ package com.google.gcloud; +import com.google.common.collect.AbstractIterator; + import java.io.Serializable; import java.util.Collections; +import java.util.Iterator; import java.util.Objects; /** @@ -35,6 +38,33 @@ public interface NextPageFetcher extends Serializable { Page nextPage(); } + static class PageIterator extends AbstractIterator { + + private Iterator currentPageIterator; + private Page currentPage; + + PageIterator(Page currentPage) { + this.currentPageIterator = currentPage.values().iterator(); + this.currentPage = currentPage; + } + + @Override + protected T computeNext() { + if (currentPageIterator.hasNext()) { + return currentPageIterator.next(); + } + Page nextPage = currentPage.nextPage(); + if (nextPage != null) { + currentPage = nextPage; + currentPageIterator = currentPage.values().iterator(); + if (currentPageIterator.hasNext()) { + return currentPageIterator.next(); + } + } + return endOfData(); + } + } + /** * Creates a {@code PageImpl} object. In order for the object to be serializable the {@code * results} parameter must be serializable. @@ -50,6 +80,11 @@ public Iterable values() { return results == null ? Collections.EMPTY_LIST : results; } + @Override + public Iterator iterateAll() { + return new PageIterator(this); + } + @Override public String nextPageCursor() { return cursor; diff --git a/gcloud-java-core/src/test/java/com/google/gcloud/PageImplTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/PageImplTest.java index 78aa3feaa281..fb289186de8d 100644 --- a/gcloud-java-core/src/test/java/com/google/gcloud/PageImplTest.java +++ b/gcloud-java-core/src/test/java/com/google/gcloud/PageImplTest.java @@ -26,21 +26,38 @@ public class PageImplTest { + private static final ImmutableList VALUES = ImmutableList.of("1", "2"); + private static final ImmutableList NEXT_VALUES = ImmutableList.of("3", "4"); + private static final ImmutableList ALL_VALUES = ImmutableList.builder() + .addAll(VALUES) + .addAll(NEXT_VALUES) + .build(); + @Test - public void testPage() throws Exception { - ImmutableList values = ImmutableList.of("1", "2"); - final PageImpl nextResult = - new PageImpl<>(null, "c", Collections.emptyList()); + public void testPage() { + final PageImpl nextResult = new PageImpl<>(null, "c", NEXT_VALUES); PageImpl.NextPageFetcher fetcher = new PageImpl.NextPageFetcher() { - @Override public PageImpl nextPage() { return nextResult; } }; - PageImpl result = new PageImpl<>(fetcher, "c", values); + PageImpl result = new PageImpl<>(fetcher, "c", VALUES); assertEquals(nextResult, result.nextPage()); assertEquals("c", result.nextPageCursor()); - assertEquals(values, ImmutableList.copyOf(result.values().iterator())); + assertEquals(VALUES, result.values()); + } + + @Test + public void testIterateAll() { + final PageImpl nextResult = new PageImpl<>(null, "c", NEXT_VALUES); + PageImpl.NextPageFetcher fetcher = new PageImpl.NextPageFetcher() { + @Override + public PageImpl nextPage() { + return nextResult; + } + }; + PageImpl result = new PageImpl<>(fetcher, "c", VALUES); + assertEquals(ALL_VALUES, ImmutableList.copyOf(result.iterateAll())); } } diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java index 7cf7fe2454fc..b7a36de7589e 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -53,6 +53,7 @@ import java.security.cert.CertificateException; import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -214,12 +215,9 @@ String parse(String... args) { public void run(Storage storage, String bucketName) { if (bucketName == null) { // list buckets - Page bucketPage = storage.list(); - while (bucketPage != null) { - for (BucketInfo b : bucketPage.values()) { - System.out.println(b); - } - bucketPage = bucketPage.nextPage(); + Iterator bucketInfoIterator = storage.list().iterateAll(); + while (bucketInfoIterator.hasNext()) { + System.out.println(bucketInfoIterator.next()); } } else { // list a bucket's blobs @@ -228,12 +226,9 @@ public void run(Storage storage, String bucketName) { System.out.println("No such bucket"); return; } - Page blobPage = bucket.list(); - while (blobPage != null) { - for (Blob b : blobPage.values()) { - System.out.println(b.info()); - } - blobPage = blobPage.nextPage(); + Iterator blobIterator = bucket.list().iterateAll(); + while (blobIterator.hasNext()) { + System.out.println(blobIterator.next().info()); } } } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java index ff6fd68fd1eb..1da45fae466e 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java @@ -33,6 +33,7 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Iterator; import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -90,6 +91,11 @@ public Page nextPage() { public Iterable values() { return BLOB_LIST; } + + @Override + public Iterator iterateAll() { + return BLOB_LIST.iterator(); + } }; private static String keyPath = "/does/not/exist/key." + UUID.randomUUID().toString() + ".json"; From c0842501db691237da0a2f0d4c466dfbe596bad1 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 6 Nov 2015 10:00:34 -0800 Subject: [PATCH 04/94] Add project ID section --- README.md | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d6da19e83ed0..3f76a2d28211 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,36 @@ Example Applications - [`StorageExample`](https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java) - A simple command line interface providing some of Cloud Storage's functionality - Read more about using this application on the [`gcloud-java-examples` docs page](http://googlecloudplatform.github.io/gcloud-java/apidocs/?com/google/gcloud/examples/StorageExample.html). +Specifying a Project +-------------------- + +Most `gcloud-java` libraries require a project ID. There are multiple ways to specify this project ID. + +1. When using `gcloud-java` libraries from within Compute/App Engine, there's no need to specify a project ID. It is automatically inferred from the production environment. +2. When using `gcloud-java` elsewhere, you can do one of the following: + * Supply the project ID when building the service options. For example, to use Datastore from a project with ID "PROJECT_ID", you can write: + + ```java + Datastore datastore = DatastoreOptions.builder().projectId("PROJECT_ID").build().service(); + ``` + * Specify the environment variable `GCLOUD_PROJECT`. For example, type the following into command line: + + ```bash + export GCLOUD_PROJECT=PROJECT_ID + ``` + * Set the project ID using the [Google Cloud SDK](https://cloud.google.com/sdk/?hl=en). To use the SDK, [download the SDK](https://cloud.google.com/sdk/?hl=en) if you haven't already, and set the project ID from the command line. For example: + + ``` + gcloud config set project PROJECT_ID + ``` + +`gcloud-java` determines the project ID from the following sources in the listed order, stopping once it finds a value: + +1. Project ID supplied when building the service options. +2. Project ID specified by the environment variable `GCLOUD_PROJECT`. +3. Project ID used by App Engine. +4. Project ID specified in the Google Cloud SDK. + Authentication -------------- @@ -45,7 +75,10 @@ There are multiple ways to authenticate to use Google Cloud services. 1. When using `gcloud-java` libraries from within Compute/App Engine, no additional authentication steps are necessary. 2. When using `gcloud-java` libraries elsewhere, there are two options: * [Generate a JSON service account key](https://cloud.google.com/storage/docs/authentication?hl=en#service_accounts). After downloading that key, you must do one of the following: - * Define the environment variable GOOGLE_APPLICATION_CREDENTIALS to be the location of the key. For example, `export GOOGLE_APPLICATION_CREDENTIALS=/path/to/my/key.json` + * Define the environment variable GOOGLE_APPLICATION_CREDENTIALS to be the location of the key. For example: + ```bash + export GOOGLE_APPLICATION_CREDENTIALS=/path/to/my/key.json + ``` * Supply the JSON credentials file when building the service options. For example, this Storage object has the necessary permissions to interact with your Google Cloud Storage data: ```java Storage storage = StorageOptions.builder() @@ -53,7 +86,7 @@ There are multiple ways to authenticate to use Google Cloud services. .build() .service(); ``` - * If running locally for development/testing, you can use use [Google Cloud SDK](https://cloud.google.com/sdk/?hl=en). To use the SDK authentication, [download the SDK](https://cloud.google.com/sdk/?hl=en) if you haven't already. Then login using the SDK (`gcloud auth login` in command line), and set your current project using `gcloud config set project PROJECT_ID`. + * If running locally for development/testing, you can use use Google Cloud SDK. Download the SDK if you haven't already, then login using the SDK (`gcloud auth login` in command line). Be sure your current project is set correctly by running `gcloud config set project PROJECT_ID`. `gcloud-java` looks for credentials in the following order, stopping once it finds credentials: @@ -63,13 +96,6 @@ There are multiple ways to authenticate to use Google Cloud services. 4. Google Cloud SDK credentials 5. Compute Engine credentials -Note that this sequence is different than the order in which `gcloud-java` determines the project ID. The project ID is determined in the following order: - -1. Project ID supplied when building the service options -2. Project ID specified by the environment variable `GCLOUD_PROJECT` -3. App Engine project ID -4. Google Cloud SDK project ID - Google Cloud Datastore ---------------------- From 67fdcc08177ec9973465da032f1c504160709f34 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 6 Nov 2015 11:00:44 -0800 Subject: [PATCH 05/94] fix order of project ID lookup, cleanup --- README.md | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3f76a2d28211..0447040de628 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ Example Applications - [`StorageExample`](https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java) - A simple command line interface providing some of Cloud Storage's functionality - Read more about using this application on the [`gcloud-java-examples` docs page](http://googlecloudplatform.github.io/gcloud-java/apidocs/?com/google/gcloud/examples/StorageExample.html). -Specifying a Project --------------------- +Specifying a Project ID +----------------------- Most `gcloud-java` libraries require a project ID. There are multiple ways to specify this project ID. @@ -49,11 +49,7 @@ Most `gcloud-java` libraries require a project ID. There are multiple ways to s ```java Datastore datastore = DatastoreOptions.builder().projectId("PROJECT_ID").build().service(); ``` - * Specify the environment variable `GCLOUD_PROJECT`. For example, type the following into command line: - - ```bash - export GCLOUD_PROJECT=PROJECT_ID - ``` + * Specify the environment variable `GCLOUD_PROJECT` to be your desired project ID. * Set the project ID using the [Google Cloud SDK](https://cloud.google.com/sdk/?hl=en). To use the SDK, [download the SDK](https://cloud.google.com/sdk/?hl=en) if you haven't already, and set the project ID from the command line. For example: ``` @@ -62,10 +58,11 @@ Most `gcloud-java` libraries require a project ID. There are multiple ways to s `gcloud-java` determines the project ID from the following sources in the listed order, stopping once it finds a value: -1. Project ID supplied when building the service options. -2. Project ID specified by the environment variable `GCLOUD_PROJECT`. -3. Project ID used by App Engine. -4. Project ID specified in the Google Cloud SDK. +1. Project ID supplied when building the service options +2. Project ID specified by the environment variable `GCLOUD_PROJECT` +3. App Engine project ID +4. Compute Engine project ID +5. Google Cloud SDK project ID Authentication -------------- @@ -86,7 +83,7 @@ There are multiple ways to authenticate to use Google Cloud services. .build() .service(); ``` - * If running locally for development/testing, you can use use Google Cloud SDK. Download the SDK if you haven't already, then login using the SDK (`gcloud auth login` in command line). Be sure your current project is set correctly by running `gcloud config set project PROJECT_ID`. + * If running locally for development/testing, you can use use Google Cloud SDK. Download the SDK if you haven't already, then login using the SDK (`gcloud auth login` in command line). Be sure to set your project ID as described above. `gcloud-java` looks for credentials in the following order, stopping once it finds credentials: From 2ce257f54600cf4c5e1342635c6eb9bf287cbc7b Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 6 Nov 2015 17:41:50 -0800 Subject: [PATCH 06/94] update auth dependency --- gcloud-java-core/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml index e849594226db..f12267db49e4 100644 --- a/gcloud-java-core/pom.xml +++ b/gcloud-java-core/pom.xml @@ -20,12 +20,12 @@ com.google.auth google-auth-library-credentials - 0.1.0 + 0.3.0 com.google.auth google-auth-library-oauth2-http - 0.1.0 + 0.3.0 com.google.http-client From e063bc5707ae1cac53f8d2ec0d3d971dd94e617e Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 6 Nov 2015 18:10:50 -0800 Subject: [PATCH 07/94] Auth cleanup --- README.md | 6 +- .../com/google/gcloud/AuthCredentials.java | 113 +----------------- .../com/google/gcloud/ServiceOptions.java | 35 +++--- .../gcloud/datastore/SerializationTest.java | 24 ++-- .../gcloud/storage/SerializationTest.java | 22 ++-- 5 files changed, 45 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index 9ba994d572c1..0833de90dfde 100644 --- a/README.md +++ b/README.md @@ -96,9 +96,9 @@ There are multiple ways to authenticate to use Google Cloud services. `gcloud-java` looks for credentials in the following order, stopping once it finds credentials: 1. Credentials supplied when building the service options -2. App Engine credentials -3. Key file pointed to by the GOOGLE_APPLICATION_CREDENTIALS environment variable -4. Google Cloud SDK credentials +2. Key file pointed to by the GOOGLE_APPLICATION_CREDENTIALS environment variable +3. Google Cloud SDK credentials +4. App Engine credentials 5. Compute Engine credentials Google Cloud Datastore diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java index 73c66279ea53..990d30eca618 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java @@ -19,12 +19,8 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.compute.ComputeCredential; -import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.auth.http.HttpCredentialsAdapter; import com.google.auth.oauth2.GoogleCredentials; @@ -32,7 +28,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.Serializable; -import java.security.GeneralSecurityException; import java.security.PrivateKey; import java.util.Objects; import java.util.Set; @@ -42,45 +37,6 @@ */ public abstract class AuthCredentials implements Restorable { - private static class AppEngineAuthCredentials extends AuthCredentials { - - private static final AuthCredentials INSTANCE = new AppEngineAuthCredentials(); - private static final AppEngineAuthCredentialsState STATE = - new AppEngineAuthCredentialsState(); - - private static class AppEngineAuthCredentialsState - implements RestorableState, Serializable { - - private static final long serialVersionUID = 3558563960848658928L; - - @Override - public AuthCredentials restore() { - return INSTANCE; - } - - @Override - public int hashCode() { - return getClass().getName().hashCode(); - } - - @Override - public boolean equals(Object obj) { - return obj instanceof AppEngineAuthCredentialsState; - } - } - - @Override - protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, - Set scopes) { - return new AppIdentityCredential(scopes); - } - - @Override - public RestorableState capture() { - return STATE; - } - } - public static class ServiceAccountAuthCredentials extends AuthCredentials { private final String account; @@ -163,55 +119,6 @@ public RestorableState capture() { } } - private static class ComputeEngineAuthCredentials extends AuthCredentials { - - private ComputeCredential computeCredential; - - private static final ComputeEngineAuthCredentialsState STATE = - new ComputeEngineAuthCredentialsState(); - - private static class ComputeEngineAuthCredentialsState - implements RestorableState, Serializable { - - private static final long serialVersionUID = -6168594072854417404L; - - @Override - public AuthCredentials restore() { - try { - return new ComputeEngineAuthCredentials(); - } catch (IOException | GeneralSecurityException e) { - throw new IllegalStateException( - "Could not restore " + ComputeEngineAuthCredentials.class.getSimpleName(), e); - } - } - - @Override - public int hashCode() { - return getClass().getName().hashCode(); - } - - @Override - public boolean equals(Object obj) { - return obj instanceof ComputeEngineAuthCredentialsState; - } - } - - ComputeEngineAuthCredentials() throws IOException, GeneralSecurityException { - computeCredential = getComputeCredential(); - } - - @Override - protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, - Set scopes) { - return computeCredential; - } - - @Override - public RestorableState capture() { - return STATE; - } - } - private static class ApplicationDefaultAuthCredentials extends AuthCredentials { private GoogleCredentials googleCredentials; @@ -264,21 +171,12 @@ public RestorableState capture() { protected abstract HttpRequestInitializer httpRequestInitializer(HttpTransport transport, Set scopes); - public static AuthCredentials createForAppEngine() { - return AppEngineAuthCredentials.INSTANCE; - } - - public static AuthCredentials createForComputeEngine() - throws IOException, GeneralSecurityException { - return new ComputeEngineAuthCredentials(); - } - /** * Returns the Application Default Credentials. * *

Returns the Application Default Credentials which are credentials that identify and * authorize the whole application. This is the built-in service account if running on - * Google Compute Engine or the credentials file can be read from the path in the environment + * Google App/Compute Engine or the credentials file can be read from the path in the environment * variable GOOGLE_APPLICATION_CREDENTIALS. *

* @@ -327,13 +225,4 @@ public static ServiceAccountAuthCredentials createForJson(InputStream jsonCreden public static AuthCredentials noCredentials() { return ServiceAccountAuthCredentials.NO_CREDENTIALS; } - - static ComputeCredential getComputeCredential() throws IOException, GeneralSecurityException { - NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); - // Try to connect using Google Compute Engine service account credentials. - ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory()); - // Force token refresh to detect if we are running on Google Compute Engine. - credential.refreshToken(); - return credential; - } } diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java index 898897833287..876317162add 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java @@ -21,10 +21,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.client.extensions.appengine.http.UrlFetchTransport; +import com.google.api.client.googleapis.compute.ComputeCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson.JacksonFactory; import com.google.common.collect.Iterables; import com.google.gcloud.spi.ServiceRpcFactory; @@ -41,6 +44,7 @@ import java.lang.reflect.Method; import java.net.HttpURLConnection; import java.net.URL; +import java.security.GeneralSecurityException; import java.util.Enumeration; import java.util.Locale; import java.util.Objects; @@ -111,12 +115,22 @@ public HttpTransport create() { } // Consider Compute try { - return AuthCredentials.getComputeCredential().getTransport(); + return getComputeHttpTransport(); } catch (Exception e) { // Maybe not on GCE } return new NetHttpTransport(); } + + private static HttpTransport getComputeHttpTransport() + throws IOException, GeneralSecurityException { + NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + // Try to connect using Google Compute Engine service account credentials. + ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory()); + // Force token refresh to detect if we are running on Google Compute Engine. + credential.refreshToken(); + return transport; + } } /** @@ -326,28 +340,11 @@ protected ServiceOptions(Class> ser } private static AuthCredentials defaultAuthCredentials() { - // Consider App Engine. This will not be needed once issue #21 is fixed. - if (appEngineAppId() != null) { - try { - return AuthCredentials.createForAppEngine(); - } catch (Exception ignore) { - // Maybe not on App Engine - } - } - try { return AuthCredentials.createApplicationDefaults(); } catch (Exception ex) { - // fallback to old-style - } - - // Consider old-style Compute. This will not be needed once issue #21 is fixed. - try { - return AuthCredentials.createForComputeEngine(); - } catch (Exception ignore) { - // Maybe not on GCE + return AuthCredentials.noCredentials(); } - return AuthCredentials.noCredentials(); } protected static String appEngineAppId() { diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 1ad690938ef5..89da268562b3 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -133,20 +133,22 @@ public class SerializationTest { @Test public void testServiceOptions() throws Exception { - DatastoreOptions options = DatastoreOptions.builder() - .authCredentials(AuthCredentials.createForAppEngine()) - .normalizeDataset(false) - .projectId("ds1") - .build(); + DatastoreOptions options = + DatastoreOptions.builder() + .authCredentials(AuthCredentials.createApplicationDefaults()) + .normalizeDataset(false) + .projectId("ds1") + .build(); DatastoreOptions serializedCopy = serializeAndDeserialize(options); assertEquals(options, serializedCopy); - options = options.toBuilder() - .namespace("ns1") - .retryParams(RetryParams.getDefaultInstance()) - .authCredentials(AuthCredentials.noCredentials()) - .force(true) - .build(); + options = + options.toBuilder() + .namespace("ns1") + .retryParams(RetryParams.getDefaultInstance()) + .authCredentials(AuthCredentials.noCredentials()) + .force(true) + .build(); serializedCopy = serializeAndDeserialize(options); assertEquals(options, serializedCopy); } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java index a125a64df6d6..e8481a2f0d90 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java @@ -73,19 +73,21 @@ public class SerializationTest { @Test public void testServiceOptions() throws Exception { - StorageOptions options = StorageOptions.builder() - .projectId("p1") - .authCredentials(AuthCredentials.createForAppEngine()) - .build(); + StorageOptions options = + StorageOptions.builder() + .projectId("p1") + .authCredentials(AuthCredentials.createApplicationDefaults()) + .build(); StorageOptions serializedCopy = serializeAndDeserialize(options); assertEquals(options, serializedCopy); - options = options.toBuilder() - .projectId("p2") - .retryParams(RetryParams.getDefaultInstance()) - .authCredentials(AuthCredentials.noCredentials()) - .pathDelimiter(":") - .build(); + options = + options.toBuilder() + .projectId("p2") + .retryParams(RetryParams.getDefaultInstance()) + .authCredentials(AuthCredentials.noCredentials()) + .pathDelimiter(":") + .build(); serializedCopy = serializeAndDeserialize(options); assertEquals(options, serializedCopy); } From 9394d2819d2c92dc3421d986846c101d5e65f7ac Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 6 Nov 2015 19:10:24 -0800 Subject: [PATCH 08/94] Remove DefaultAuthCredentials from serialization tests, since there are no default auth credentials on Travis --- .../test/java/com/google/gcloud/datastore/SerializationTest.java | 1 - .../test/java/com/google/gcloud/storage/SerializationTest.java | 1 - 2 files changed, 2 deletions(-) diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 89da268562b3..129122c8643d 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -135,7 +135,6 @@ public class SerializationTest { public void testServiceOptions() throws Exception { DatastoreOptions options = DatastoreOptions.builder() - .authCredentials(AuthCredentials.createApplicationDefaults()) .normalizeDataset(false) .projectId("ds1") .build(); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java index e8481a2f0d90..a0bfa0213796 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java @@ -76,7 +76,6 @@ public void testServiceOptions() throws Exception { StorageOptions options = StorageOptions.builder() .projectId("p1") - .authCredentials(AuthCredentials.createApplicationDefaults()) .build(); StorageOptions serializedCopy = serializeAndDeserialize(options); assertEquals(options, serializedCopy); From 076127d6ce599716519eeffd5595fe8c5313b5df Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Sat, 7 Nov 2015 23:50:45 +0100 Subject: [PATCH 09/94] Add loop for empty pages in PageImpl.iterateAll --- .../main/java/com/google/gcloud/PageImpl.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java b/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java index 5f70b22ec577..5e83b53b33a9 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java @@ -50,18 +50,14 @@ static class PageIterator extends AbstractIterator { @Override protected T computeNext() { - if (currentPageIterator.hasNext()) { - return currentPageIterator.next(); - } - Page nextPage = currentPage.nextPage(); - if (nextPage != null) { - currentPage = nextPage; - currentPageIterator = currentPage.values().iterator(); - if (currentPageIterator.hasNext()) { - return currentPageIterator.next(); + while (!currentPageIterator.hasNext()) { + currentPage = currentPage.nextPage(); + if (currentPage == null) { + return endOfData(); } + currentPageIterator = currentPage.values().iterator(); } - return endOfData(); + return currentPageIterator.next(); } } From 607460518d477a235ebcd8afd155b5c35c220fd2 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Mon, 9 Nov 2015 08:19:46 -0800 Subject: [PATCH 10/94] Modify project id lookup order and revert style changes --- README.md | 4 +- .../com/google/gcloud/ServiceOptions.java | 67 ++++++++++--------- .../gcloud/datastore/SerializationTest.java | 22 +++--- .../gcloud/storage/SerializationTest.java | 20 +++--- 4 files changed, 55 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 0833de90dfde..9cdee74ee8f8 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,8 @@ Most `gcloud-java` libraries require a project ID. There are multiple ways to s 1. Project ID supplied when building the service options 2. Project ID specified by the environment variable `GCLOUD_PROJECT` 3. App Engine project ID -4. Compute Engine project ID -5. Google Cloud SDK project ID +4. Google Cloud SDK project ID +5. Compute Engine project ID Authentication -------------- diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java index 876317162add..298a1223acac 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java @@ -364,19 +364,6 @@ protected String defaultProject() { } protected static String googleCloudProjectId() { - try { - URL url = new URL("http://metadata/computeMetadata/v1/project/project-id"); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setRequestProperty("X-Google-Metadata-Request", "True"); - InputStream input = connection.getInputStream(); - if (connection.getResponseCode() == 200) { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, UTF_8))) { - return reader.readLine(); - } - } - } catch (IOException ignore) { - // ignore - } File configDir; if (System.getenv().containsKey("CLOUDSDK_CONFIG")) { configDir = new File(System.getenv("CLOUDSDK_CONFIG")); @@ -385,38 +372,52 @@ protected static String googleCloudProjectId() { } else { configDir = new File(System.getProperty("user.home"), ".config/gcloud"); } - FileReader fileReader; + FileReader fileReader = null; try { fileReader = new FileReader(new File(configDir, "configurations/config_default")); } catch (FileNotFoundException newConfigFileNotFoundEx) { try { fileReader = new FileReader(new File(configDir, "properties")); } catch (FileNotFoundException oldConfigFileNotFoundEx) { - // return null if we can't find config file - return null; + // ignore } } - try (BufferedReader reader = new BufferedReader(fileReader)) { - String line; - String section = null; - Pattern projectPattern = Pattern.compile("^project\\s*=\\s*(.*)$"); - Pattern sectionPattern = Pattern.compile("^\\[(.*)\\]$"); - while ((line = reader.readLine()) != null) { - if (line.isEmpty() || line.startsWith(";")) { - continue; - } - line = line.trim(); - Matcher matcher = sectionPattern.matcher(line); - if (matcher.matches()) { - section = matcher.group(1); - } else if (section == null || section.equals("core")) { - matcher = projectPattern.matcher(line); + if (fileReader != null) { + try (BufferedReader reader = new BufferedReader(fileReader)) { + String line; + String section = null; + Pattern projectPattern = Pattern.compile("^project\\s*=\\s*(.*)$"); + Pattern sectionPattern = Pattern.compile("^\\[(.*)\\]$"); + while ((line = reader.readLine()) != null) { + if (line.isEmpty() || line.startsWith(";")) { + continue; + } + line = line.trim(); + Matcher matcher = sectionPattern.matcher(line); if (matcher.matches()) { - return matcher.group(1); + section = matcher.group(1); + } else if (section == null || section.equals("core")) { + matcher = projectPattern.matcher(line); + if (matcher.matches()) { + return matcher.group(1); + } } } + } catch (IOException ex) { + // ignore } - } catch (IOException ex) { + } + try { + URL url = new URL("http://metadata/computeMetadata/v1/project/project-id"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("X-Google-Metadata-Request", "True"); + InputStream input = connection.getInputStream(); + if (connection.getResponseCode() == 200) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, UTF_8))) { + return reader.readLine(); + } + } + } catch (IOException ignore) { // ignore } // return null if can't determine diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 129122c8643d..48259f7a1395 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -133,21 +133,19 @@ public class SerializationTest { @Test public void testServiceOptions() throws Exception { - DatastoreOptions options = - DatastoreOptions.builder() - .normalizeDataset(false) - .projectId("ds1") - .build(); + DatastoreOptions options = DatastoreOptions.builder() + .normalizeDataset(false) + .projectId("ds1") + .build(); DatastoreOptions serializedCopy = serializeAndDeserialize(options); assertEquals(options, serializedCopy); - options = - options.toBuilder() - .namespace("ns1") - .retryParams(RetryParams.getDefaultInstance()) - .authCredentials(AuthCredentials.noCredentials()) - .force(true) - .build(); + options = options.toBuilder() + .namespace("ns1") + .retryParams(RetryParams.getDefaultInstance()) + .authCredentials(AuthCredentials.noCredentials()) + .force(true) + .build(); serializedCopy = serializeAndDeserialize(options); assertEquals(options, serializedCopy); } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java index a0bfa0213796..6ed8046e35a2 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java @@ -73,20 +73,18 @@ public class SerializationTest { @Test public void testServiceOptions() throws Exception { - StorageOptions options = - StorageOptions.builder() - .projectId("p1") - .build(); + StorageOptions options = StorageOptions.builder() + .projectId("p1") + .build(); StorageOptions serializedCopy = serializeAndDeserialize(options); assertEquals(options, serializedCopy); - options = - options.toBuilder() - .projectId("p2") - .retryParams(RetryParams.getDefaultInstance()) - .authCredentials(AuthCredentials.noCredentials()) - .pathDelimiter(":") - .build(); + options = options.toBuilder() + .projectId("p2") + .retryParams(RetryParams.getDefaultInstance()) + .authCredentials(AuthCredentials.noCredentials()) + .pathDelimiter(":") + .build(); serializedCopy = serializeAndDeserialize(options); assertEquals(options, serializedCopy); } From b2e11319cd3263b9489d47327462586660a49ea3 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 10 Nov 2015 09:09:24 -0800 Subject: [PATCH 11/94] scope appliation default credentials --- .../src/main/java/com/google/gcloud/AuthCredentials.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java index 73c66279ea53..800fcf340689 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java @@ -252,7 +252,7 @@ public boolean equals(Object obj) { @Override protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport, Set scopes) { - return new HttpCredentialsAdapter(googleCredentials); + return new HttpCredentialsAdapter(googleCredentials.createScoped(scopes)); } @Override From d3db640dcf383bced4618f0d3e52f974d4e09690 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Tue, 10 Nov 2015 09:45:33 -0800 Subject: [PATCH 12/94] Revert 2de9135149e57cec18b070028452db1efef03125 --- gcloud-java-core/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml index f12267db49e4..e849594226db 100644 --- a/gcloud-java-core/pom.xml +++ b/gcloud-java-core/pom.xml @@ -20,12 +20,12 @@ com.google.auth google-auth-library-credentials - 0.3.0 + 0.1.0 com.google.auth google-auth-library-oauth2-http - 0.3.0 + 0.1.0 com.google.http-client From ad12f4143bbcb5d6557055452717c7380cf54959 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Tue, 10 Nov 2015 12:59:34 -0800 Subject: [PATCH 13/94] Allow services to avoid setting project ID --- .../java/com/google/gcloud/ServiceOptions.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java index 898897833287..d91b3c35c8dd 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java @@ -306,7 +306,10 @@ public B readTimeout(int readTimeout) { protected ServiceOptions(Class> serviceFactoryClass, Class> rpcFactoryClass, Builder builder) { - projectId = checkNotNull(builder.projectId != null ? builder.projectId : defaultProject()); + projectId = builder.projectId != null ? builder.projectId : defaultProject(); + if (projectIdRequired()) { + checkNotNull(projectId); + } host = firstNonNull(builder.host, defaultHost()); httpTransportFactory = firstNonNull(builder.httpTransportFactory, getFromServiceLoader(HttpTransportFactory.class, DefaultHttpTransportFactory.INSTANCE)); @@ -325,6 +328,16 @@ protected ServiceOptions(Class> ser clock = firstNonNull(builder.clock, Clock.defaultClock()); } + /** + * Returns whether a service requires a project ID. This method may be overridden in + * service-specific Options objects. + * + * @return true if a project ID is required to use the service, false if not. + */ + public boolean projectIdRequired() { + return true; + } + private static AuthCredentials defaultAuthCredentials() { // Consider App Engine. This will not be needed once issue #21 is fixed. if (appEngineAppId() != null) { @@ -462,6 +475,8 @@ public ServiceRpcT rpc() { /** * Returns the project id. + * + * Return value can be null (for services that don't require a project id). */ public String projectId() { return projectId; From 93c834e954e7ce595d86b8fd45a9f2851dfcad4a Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Tue, 10 Nov 2015 13:18:01 -0800 Subject: [PATCH 14/94] Update version to 0.0.11 in preparation for 0.0.11 release --- gcloud-java-core/pom.xml | 2 +- gcloud-java-datastore/pom.xml | 2 +- gcloud-java-examples/pom.xml | 2 +- gcloud-java-storage/pom.xml | 2 +- gcloud-java/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml index e849594226db..0dd04b1e837d 100644 --- a/gcloud-java-core/pom.xml +++ b/gcloud-java-core/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.11-SNAPSHOT + 0.0.11 gcloud-java-core diff --git a/gcloud-java-datastore/pom.xml b/gcloud-java-datastore/pom.xml index c7c57d91ae1d..b576ce47fdc8 100644 --- a/gcloud-java-datastore/pom.xml +++ b/gcloud-java-datastore/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.11-SNAPSHOT + 0.0.11 gcloud-java-datastore diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml index 55548e756be3..b1d3a3f27c2b 100644 --- a/gcloud-java-examples/pom.xml +++ b/gcloud-java-examples/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.11-SNAPSHOT + 0.0.11 gcloud-java-examples diff --git a/gcloud-java-storage/pom.xml b/gcloud-java-storage/pom.xml index 82b67277f4fe..910aab0e35f7 100644 --- a/gcloud-java-storage/pom.xml +++ b/gcloud-java-storage/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.11-SNAPSHOT + 0.0.11 gcloud-java-storage diff --git a/gcloud-java/pom.xml b/gcloud-java/pom.xml index 6ad0fce8f1b1..1ccb13fcb214 100644 --- a/gcloud-java/pom.xml +++ b/gcloud-java/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.11-SNAPSHOT + 0.0.11 diff --git a/pom.xml b/pom.xml index e35a620247d1..df842cdb9bbc 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.gcloud gcloud-java-pom pom - 0.0.11-SNAPSHOT + 0.0.11 GCloud Java https://github.com/GoogleCloudPlatform/gcloud-java From ffbfba48a6426ff63c08ff2117e58681f251fbf2 Mon Sep 17 00:00:00 2001 From: travis-ci Date: Tue, 10 Nov 2015 22:15:04 +0000 Subject: [PATCH 15/94] Updating version in README files. --- README.md | 6 +++--- gcloud-java-core/README.md | 6 +++--- gcloud-java-datastore/README.md | 6 +++--- gcloud-java-examples/README.md | 6 +++--- gcloud-java-storage/README.md | 6 +++--- gcloud-java/README.md | 6 +++--- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 9ba994d572c1..0fd576a6a127 100644 --- a/README.md +++ b/README.md @@ -25,16 +25,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java - 0.0.10 + 0.0.11 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java:jar:0.0.10' +compile 'com.google.gcloud:gcloud-java:jar:0.0.11' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.10" +libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.11" ``` Example Applications diff --git a/gcloud-java-core/README.md b/gcloud-java-core/README.md index 032127540d19..f84fb33993e7 100644 --- a/gcloud-java-core/README.md +++ b/gcloud-java-core/README.md @@ -17,16 +17,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java-core - 0.0.10 + 0.0.11 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java-core:jar:0.0.10' +compile 'com.google.gcloud:gcloud-java-core:jar:0.0.11' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java-core" % "0.0.10" +libraryDependencies += "com.google.gcloud" % "gcloud-java-core" % "0.0.11" ``` Java Versions diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md index 8915f2d37a55..67483fae2b6e 100644 --- a/gcloud-java-datastore/README.md +++ b/gcloud-java-datastore/README.md @@ -20,16 +20,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java-datastore - 0.0.10 + 0.0.11 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java-datastore:jar:0.0.10' +compile 'com.google.gcloud:gcloud-java-datastore:jar:0.0.11' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java-datastore" % "0.0.10" +libraryDependencies += "com.google.gcloud" % "gcloud-java-datastore" % "0.0.11" ``` Example Application diff --git a/gcloud-java-examples/README.md b/gcloud-java-examples/README.md index 9afe16a2b1a5..bc738de41b51 100644 --- a/gcloud-java-examples/README.md +++ b/gcloud-java-examples/README.md @@ -17,16 +17,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java-examples - 0.0.10 + 0.0.11 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java-examples:jar:0.0.10' +compile 'com.google.gcloud:gcloud-java-examples:jar:0.0.11' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java-examples" % "0.0.10" +libraryDependencies += "com.google.gcloud" % "gcloud-java-examples" % "0.0.11" ``` To run examples from your command line: diff --git a/gcloud-java-storage/README.md b/gcloud-java-storage/README.md index 8722da76cec4..f2b99388ff0f 100644 --- a/gcloud-java-storage/README.md +++ b/gcloud-java-storage/README.md @@ -20,16 +20,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java-storage - 0.0.10 + 0.0.11 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java-storage:jar:0.0.10' +compile 'com.google.gcloud:gcloud-java-storage:jar:0.0.11' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java-storage" % "0.0.10" +libraryDependencies += "com.google.gcloud" % "gcloud-java-storage" % "0.0.11" ``` Example Application diff --git a/gcloud-java/README.md b/gcloud-java/README.md index baa1e5c53b1c..eaaed21af5fe 100644 --- a/gcloud-java/README.md +++ b/gcloud-java/README.md @@ -25,16 +25,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java - 0.0.10 + 0.0.11 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java:jar:0.0.10' +compile 'com.google.gcloud:gcloud-java:jar:0.0.11' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.10" +libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.11" ``` Java Versions From fc72ea0b0e246921c491f8460cd9ce7dab94e82c Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Tue, 10 Nov 2015 14:35:11 -0800 Subject: [PATCH 16/94] Update version to 0.0.12-SNAPSHOT --- gcloud-java-core/pom.xml | 2 +- gcloud-java-datastore/pom.xml | 2 +- gcloud-java-examples/pom.xml | 2 +- gcloud-java-storage/pom.xml | 2 +- gcloud-java/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml index 0dd04b1e837d..e13933bd2beb 100644 --- a/gcloud-java-core/pom.xml +++ b/gcloud-java-core/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.11 + 0.0.12-SNAPSHOT gcloud-java-core diff --git a/gcloud-java-datastore/pom.xml b/gcloud-java-datastore/pom.xml index b576ce47fdc8..b58e9e0ffc74 100644 --- a/gcloud-java-datastore/pom.xml +++ b/gcloud-java-datastore/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.11 + 0.0.12-SNAPSHOT gcloud-java-datastore diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml index b1d3a3f27c2b..c461846acab2 100644 --- a/gcloud-java-examples/pom.xml +++ b/gcloud-java-examples/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.11 + 0.0.12-SNAPSHOT gcloud-java-examples diff --git a/gcloud-java-storage/pom.xml b/gcloud-java-storage/pom.xml index 910aab0e35f7..ef3ddec79816 100644 --- a/gcloud-java-storage/pom.xml +++ b/gcloud-java-storage/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.11 + 0.0.12-SNAPSHOT gcloud-java-storage diff --git a/gcloud-java/pom.xml b/gcloud-java/pom.xml index 1ccb13fcb214..d26c38f517d5 100644 --- a/gcloud-java/pom.xml +++ b/gcloud-java/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.11 + 0.0.12-SNAPSHOT diff --git a/pom.xml b/pom.xml index df842cdb9bbc..78cbfe11d351 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.gcloud gcloud-java-pom pom - 0.0.11 + 0.0.12-SNAPSHOT GCloud Java https://github.com/GoogleCloudPlatform/gcloud-java From 6c408256e1621c7e0f3640fb7a4ffdef139f5241 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Tue, 10 Nov 2015 15:02:44 -0800 Subject: [PATCH 17/94] Make projectIdRequired() method protected. Also supply nice error message if project could not be determined and is required --- .../src/main/java/com/google/gcloud/ServiceOptions.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java index d91b3c35c8dd..0793470ade83 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java @@ -17,7 +17,7 @@ package com.google.gcloud; import static com.google.common.base.MoreObjects.firstNonNull; -import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkArgument; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.client.extensions.appengine.http.UrlFetchTransport; @@ -308,7 +308,10 @@ protected ServiceOptions(Class> ser Builder builder) { projectId = builder.projectId != null ? builder.projectId : defaultProject(); if (projectIdRequired()) { - checkNotNull(projectId); + checkArgument( + projectId != null, + "A project ID is required for this service but could not be determined from the builder or " + + "the environment. Please set a project ID using the builder."); } host = firstNonNull(builder.host, defaultHost()); httpTransportFactory = firstNonNull(builder.httpTransportFactory, @@ -334,7 +337,7 @@ protected ServiceOptions(Class> ser * * @return true if a project ID is required to use the service, false if not. */ - public boolean projectIdRequired() { + protected boolean projectIdRequired() { return true; } From bfd54bdefbf2e8b79b1e96d71c7adbd05eda2e7a Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 10 Nov 2015 16:37:56 -0800 Subject: [PATCH 18/94] replace rewrite with copy in the javadoc --- .../src/main/java/com/google/gcloud/storage/CopyWriter.java | 4 ++-- .../src/main/java/com/google/gcloud/storage/Storage.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java index 142f8d4b6de7..5f2632b2acde 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java @@ -79,14 +79,14 @@ public long blobSize() { } /** - * Returns {@code true} of blob rewrite finished, {@code false} otherwise. + * Returns {@code true} if blob copy has finished, {@code false} otherwise. */ public boolean isDone() { return rewriteResponse.isDone; } /** - * Returns the number of bytes copied. + * Returns the number of bytes copied. */ public long totalBytesCopied() { return rewriteResponse.totalBytesRewritten; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 52387310de71..115449064c66 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -804,7 +804,7 @@ private CopyRequest(Builder builder) { } /** - * Returns the blob to rewrite, as a {@link BlobId}. + * Returns the blob to copy, as a {@link BlobId}. */ public BlobId source() { return source; @@ -818,7 +818,7 @@ public List sourceOptions() { } /** - * Returns the rewrite target. + * Returns the {@link BlobInfo} for the target blob. */ public BlobInfo target() { return target; From dde30aaf085b0cf3a6615af0495309b2596e488f Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 15:32:36 +0100 Subject: [PATCH 19/94] Add javadoc to Acl --- .../java/com/google/gcloud/storage/Acl.java | 74 ++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Acl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Acl.java index 3d9731352400..fd75e10d92fa 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Acl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Acl.java @@ -23,7 +23,10 @@ import java.util.Objects; /** - * Access Control List on for buckets or blobs. + * Access Control List for buckets or blobs. + * + * @see + * About Access Control Lists */ public final class Acl implements Serializable { @@ -36,6 +39,9 @@ public enum Role { OWNER, READER, WRITER } + /** + * Base class for Access Control List entities. + */ public abstract static class Entity implements Serializable { private static final long serialVersionUID = -2707407252771255840L; @@ -52,10 +58,16 @@ public enum Type { this.value = value; } + /** + * Returns the type of entity. + */ public Type type() { return type; } + /** + * Returns the entity's value. + */ protected String value() { return value; } @@ -112,42 +124,75 @@ static Entity fromPb(String entity) { } } + /** + * Class for ACL Domain entities. + */ public static final class Domain extends Entity { private static final long serialVersionUID = -3033025857280447253L; + /** + * Creates a domain entity. + * + * @param domain the domain associated to this entity + */ public Domain(String domain) { super(Type.DOMAIN, domain); } + /** + * Returns the domain associated to this entity. + */ public String domain() { return value(); } } + /** + * Class for ACL Group entities. + */ public static final class Group extends Entity { private static final long serialVersionUID = -1660987136294408826L; + /** + * Creates a group entity. + * + * @param email the group email + */ public Group(String email) { super(Type.GROUP, email); } + /** + * Returns the group email. + */ public String email() { return value(); } } + /** + * Class for ACL User entities. + */ public static final class User extends Entity { private static final long serialVersionUID = 3076518036392737008L; private static final String ALL_USERS = "allUsers"; private static final String ALL_AUTHENTICATED_USERS = "allAuthenticatedUsers"; + /** + * Creates a user entity. + * + * @param email the user email + */ public User(String email) { super(Type.USER, email); } + /** + * Returns the user email. + */ public String email() { return value(); } @@ -174,6 +219,9 @@ public static User ofAllAuthenticatedUsers() { } } + /** + * Class for ACL Project entities. + */ public static final class Project extends Entity { private static final long serialVersionUID = 7933776866530023027L; @@ -185,16 +233,28 @@ public enum ProjectRole { OWNERS, EDITORS, VIEWERS } + /** + * Creates a project entity. + * + * @param pRole a role in the project, used to select project's teams + * @param projectId id of the project + */ public Project(ProjectRole pRole, String projectId) { super(Type.PROJECT, pRole.name().toLowerCase() + "-" + projectId); this.pRole = pRole; this.projectId = projectId; } + /** + * Returns the role in the project for this entity. + */ public ProjectRole projectRole() { return pRole; } + /** + * Returns the project id for this entity. + */ public String projectId() { return projectId; } @@ -214,15 +274,27 @@ String toPb() { } } + /** + * Creats an ACL object. + * + * @param entity the entity for this ACL object + * @param role the role to associate to the {@code entity} object + */ public Acl(Entity entity, Role role) { this.entity = entity; this.role = role; } + /** + * Returns the entity for this ACL object. + */ public Entity entity() { return entity; } + /** + * Returns the role associated to the entity in this ACL object. + */ public Role role() { return role; } From 93dd0c244fdbc6843ed2fc43912fa55a4c9e6258 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 15:34:49 +0100 Subject: [PATCH 20/94] Add javadoc to Blob --- .../java/com/google/gcloud/storage/Blob.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index d35fcef026c8..a8e315be0e45 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -50,6 +50,9 @@ public final class Blob { private final Storage storage; private final BlobInfo info; + /** + * Class for specifying blob source options when {@code Blob} methods are used. + */ public static class BlobSourceOption extends Option { private static final long serialVersionUID = 214616862061934846L; @@ -88,18 +91,34 @@ private Storage.BlobGetOption toGetOption(BlobInfo blobInfo) { } } + /** + * Returns an option for blob's generation match. If this option is used the request will fail + * if generation does not match. + */ public static BlobSourceOption generationMatch() { return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_MATCH); } + /** + * Returns an option for blob's generation mismatch. If this option is used the request will + * fail if generation matches. + */ public static BlobSourceOption generationNotMatch() { return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH); } + /** + * Returns an option for blob's metageneration match. If this option is used the request will + * fail if metageneration does not match. + */ public static BlobSourceOption metagenerationMatch() { return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH); } + /** + * Returns an option for blob's metageneration mismatch. If this option is used the request will + * fail if metageneration matches. + */ public static BlobSourceOption metagenerationNotMatch() { return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH); } From 6de12f20c202386d66588059b6c9ec55ff47a058 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 15:35:36 +0100 Subject: [PATCH 21/94] Add javadoc to BlobId --- .../main/java/com/google/gcloud/storage/BlobId.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobId.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobId.java index eafebe09a4cb..d1209826cc3e 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobId.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobId.java @@ -38,10 +38,16 @@ private BlobId(String bucket, String name) { this.name = name; } + /** + * Returns the name of the bucket containing the blob. + */ public String bucket() { return bucket; } + /** + * Returns the name of the blob. + */ public String name() { return name; } @@ -72,6 +78,12 @@ StorageObject toPb() { return storageObject; } + /** + * Creates a blob identifier. + * + * @param bucket the name of the bucket that contains the blob + * @param name the name of the blob + */ public static BlobId of(String bucket, String name) { return new BlobId(checkNotNull(bucket), checkNotNull(name)); } From e2a4500e57e7dd458223cff5020e990539795828 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 15:37:47 +0100 Subject: [PATCH 22/94] Add javadoc to Bucket --- .../main/java/com/google/gcloud/storage/Bucket.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java index 7c386373995f..21aafd92b5d4 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java @@ -122,6 +122,9 @@ public boolean equals(Object obj) { } } + /** + * Class for specifying bucket source options when {@code Bucket} methods are used. + */ public static class BucketSourceOption extends Option { private static final long serialVersionUID = 6928872234155522371L; @@ -152,10 +155,18 @@ private Storage.BucketGetOption toGetOption(BucketInfo bucketInfo) { } } + /** + * Returns an option for bucket's metageneration match. If this option is used the request will + * fail if metageneration does not match. + */ public static BucketSourceOption metagenerationMatch() { return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH); } + /** + * Returns an option for bucket's metageneration mismatch. If this option is used the request + * will fail if metageneration matches. + */ public static BucketSourceOption metagenerationNotMatch() { return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH); } From 9608e61fe5228de7c4faddb9e9c4a850eda69611 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 15:40:43 +0100 Subject: [PATCH 23/94] Add javadoc to Cors --- .../java/com/google/gcloud/storage/Cors.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Cors.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Cors.java index a94359f17a79..bcbbd1030dbc 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Cors.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Cors.java @@ -33,6 +33,9 @@ /** * Cross-Origin Resource Sharing (CORS) configuration for a bucket. + * + * @see + * Cross-Origin Resource Sharing (CORS) */ public final class Cors implements Serializable { @@ -57,6 +60,9 @@ public Bucket.Cors apply(Cors cors) { private final ImmutableList origins; private final ImmutableList responseHeaders; + /** + * Class for a CORS origin. + */ public static final class Origin implements Serializable { private static final long serialVersionUID = -4447958124895577993L; @@ -69,10 +75,16 @@ private Origin(String value) { this.value = checkNotNull(value); } + /** + * Returns an {@code Origin} object for all possible origins. + */ public static Origin any() { return ANY; } + /** + * Returns an {@code Origin} object for the given scheme, host and port. + */ public static Origin of(String scheme, String host, int port) { try { return of(new URI(scheme, null, host, port, null, null, null).toString()); @@ -81,6 +93,9 @@ public static Origin of(String scheme, String host, int port) { } } + /** + * Creates an {@code Origin} object for the provided value. + */ public static Origin of(String value) { if (ANY_URI.equals(value)) { return any(); @@ -111,6 +126,9 @@ public String value() { } } + /** + * CORS configuration builder. + */ public static final class Builder { private Integer maxAgeSeconds; @@ -120,26 +138,42 @@ public static final class Builder { private Builder() {} + /** + * Sets the max time in seconds in which a client can issue requests before sending a new + * preflight request. + */ public Builder maxAgeSeconds(Integer maxAgeSeconds) { this.maxAgeSeconds = maxAgeSeconds; return this; } + /** + * Sets the HTTP methods supported by this CORS configuration. + */ public Builder methods(Iterable methods) { this.methods = methods != null ? ImmutableList.copyOf(methods) : null; return this; } + /** + * Sets the origins for this CORS configuration. + */ public Builder origins(Iterable origins) { this.origins = origins != null ? ImmutableList.copyOf(origins) : null; return this; } + /** + * Sets the response headers supported by this CORS configuration. + */ public Builder responseHeaders(Iterable headers) { this.responseHeaders = headers != null ? ImmutableList.copyOf(headers) : null; return this; } + /** + * Creates a CORS configuration. + */ public Cors build() { return new Cors(this); } @@ -152,22 +186,38 @@ private Cors(Builder builder) { this.responseHeaders = builder.responseHeaders; } + /** + * Returns the max time in seconds in which a client can issue requests before sending a new + * preflight request. + */ public Integer maxAgeSeconds() { return maxAgeSeconds; } + /** + * Returns the HTTP methods supported by this CORS configuration. + */ public List methods() { return methods; } + /** + * Returns the origins in this CORS configuration. + */ public List origins() { return origins; } + /** + * Returns the response headers supported by this CORS configuration. + */ public List responseHeaders() { return responseHeaders; } + /** + * Returns a builder for this CORS configuration. + */ public Builder toBuilder() { return builder() .maxAgeSeconds(maxAgeSeconds) @@ -193,6 +243,9 @@ public boolean equals(Object obj) { && Objects.equals(responseHeaders, other.responseHeaders); } + /** + * Returns a CORS configuration builder. + */ public static Builder builder() { return new Builder(); } From 67bfcdd00e1c475eb74f222d1ea59ef7b0b3e130 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 14:31:53 +0100 Subject: [PATCH 24/94] Switch from enum to String for location and storageClass in BucketInfo, add javadoc --- .../com/google/gcloud/storage/BucketInfo.java | 330 ++++++++++-------- .../google/gcloud/storage/BucketInfoTest.java | 16 +- 2 files changed, 187 insertions(+), 159 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java index 5d69c54e0d96..3f21f74dcd66 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -80,9 +80,15 @@ public com.google.api.services.storage.model.Bucket apply(BucketInfo bucketInfo) private final List cors; private final List acl; private final List defaultAcl; - private final Location location; - private final StorageClass storageClass; - + private final String location; + private final String storageClass; + + /** + * Base class for bucket's delete rules. Allows to configure automatic deletion of blobs and blobs + * versions. + * + * @see Object Lifecycle Management + */ public abstract static class DeleteRule implements Serializable { private static final long serialVersionUID = 3137971668395933033L; @@ -153,11 +159,23 @@ static DeleteRule fromPb(Rule rule) { } } + /** + * Delete rule class that sets a Time To Live for blobs in the bucket. + * + * @see Object Lifecycle Management + */ public static class AgeDeleteRule extends DeleteRule { private static final long serialVersionUID = 5697166940712116380L; private final int daysToLive; + /** + * Creates an {@code AgeDeleteRule} object. + * + * @param daysToLive blobs' Time To Live expressed in days. The time when the age condition is + * considered to be satisfied is computed by adding {@code daysToLive} days to the + * midnight following blob's creation time in UTC. + */ public AgeDeleteRule(int daysToLive) { super(Type.AGE); this.daysToLive = daysToLive; @@ -205,11 +223,22 @@ Rule toPb() { } } + /** + * Delete rule class for blobs in the bucket that have been created before a certain date. + * + * @see Object Lifecycle Management + */ public static class CreatedBeforeDeleteRule extends DeleteRule { private static final long serialVersionUID = 881692650279195867L; private final long timeMillis; + /** + * Creates an {@code CreatedBeforeDeleteRule} object. + * + * @param timeMillis a date in UTC. Blobs that have been created before midnight of the provided + * date meet the delete condition + */ public CreatedBeforeDeleteRule(long timeMillis) { super(Type.CREATE_BEFORE); this.timeMillis = timeMillis; @@ -225,11 +254,23 @@ void populateCondition(Rule.Condition condition) { } } + /** + * Delete rule class for versioned blobs. Specifies when to delete a blob's version according to + * the number of available newer versions for that blob. + * + * @see Object Lifecycle Management + */ public static class NumNewerVersionsDeleteRule extends DeleteRule { private static final long serialVersionUID = -1955554976528303894L; private final int numNewerVersions; + /** + * Creates an {@code NumNewerVersionsDeleteRule} object. + * + * @param numNewerVersions the number of newer versions. A blob's version meets the delete + * condition when {@code numNewerVersions} newer versions are available. + */ public NumNewerVersionsDeleteRule(int numNewerVersions) { super(Type.NUM_NEWER_VERSIONS); this.numNewerVersions = numNewerVersions; @@ -245,11 +286,22 @@ void populateCondition(Rule.Condition condition) { } } + /** + * Delete rule class to distinguish between live and archived blobs. + * + * @see Object Lifecycle Management + */ public static class IsLiveDeleteRule extends DeleteRule { private static final long serialVersionUID = -3502994563121313364L; private final boolean isLive; + /** + * Creates an {@code IsLiveDeleteRule} object. + * + * @param isLive if set to {@code true} live blobs meet the delete condition. If set to + * {@code false} delete condition is met by archived objects. + */ public IsLiveDeleteRule(boolean isLive) { super(Type.IS_LIVE); this.isLive = isLive; @@ -265,134 +317,6 @@ void populateCondition(Rule.Condition condition) { } } - public static final class StorageClass implements Serializable { - - private static final long serialVersionUID = 374002156285326563L; - private static final ImmutableMap STRING_TO_OPTION; - private static final StorageClass NULL_VALUE = - new StorageClass(Data.nullOf(String.class)); - - private final String value; - - public enum Option { - DURABLE_REDUCED_AVAILABILITY, STANDARD; - - private final StorageClass storageClass; - - Option() { - storageClass = new StorageClass(name()); - } - } - - private StorageClass(String value) { - this.value = checkNotNull(value); - } - - @Override - public String toString() { - return value(); - } - - public String value() { - return value; - } - - public static StorageClass standard() { - return Option.STANDARD.storageClass; - } - - public static StorageClass durableReducedAvailability() { - return Option.DURABLE_REDUCED_AVAILABILITY.storageClass; - } - - public static StorageClass of(String value) { - Option option = STRING_TO_OPTION.get(value.toUpperCase()); - return option == null ? new StorageClass(value) : option.storageClass; - } - - static { - ImmutableMap.Builder map = ImmutableMap.builder(); - for (Option option : Option.values()) { - map.put(option.name(), option); - } - STRING_TO_OPTION = map.build(); - } - } - - public static final class Location implements Serializable { - - private static final long serialVersionUID = 9073107666838637662L; - private static final ImmutableMap STRING_TO_OPTION; - private static final Location NULL_VALUE = new Location(Data.nullOf(String.class)); - - private final String value; - - public enum Option { - US, EU, ASIA; - - private final Location location; - - Option() { - location = new Location(name()); - } - } - - private Location(String value) { - this.value = checkNotNull(value); - } - - @Override - public int hashCode() { - return Objects.hash(value); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final Location other = (Location) obj; - return Objects.equals(this.value, other.value); - } - - @Override - public String toString() { - return value(); - } - - public String value() { - return value; - } - - public static Location us() { - return Option.US.location; - } - - public static Location eu() { - return Option.EU.location; - } - - public static Location asia() { - return Option.ASIA.location; - } - - public static Location of(String value) { - Option option = STRING_TO_OPTION.get(value.toUpperCase()); - return option == null ? new Location(value) : option.location; - } - - static { - ImmutableMap.Builder map = ImmutableMap.builder(); - for (Option option : Option.values()) { - map.put(option.name(), option); - } - STRING_TO_OPTION = map.build(); - } - } - public static final class Builder { private String id; @@ -403,8 +327,8 @@ public static final class Builder { private String indexPage; private String notFoundPage; private ImmutableList deleteRules; - private StorageClass storageClass; - private Location location; + private String storageClass; + private String location; private String etag; private Long createTime; private Long metageneration; @@ -414,6 +338,9 @@ public static final class Builder { private Builder() {} + /** + * Sets the bucket's name. + */ public Builder name(String name) { this.name = checkNotNull(name); return this; @@ -434,33 +361,57 @@ Builder selfLink(String selfLink) { return this; } + /** + * Sets whether blob versioning should be enabled or not for the bucket. + */ public Builder versioningEnabled(Boolean enable) { this.versioningEnabled = firstNonNull(enable, Data.nullOf(Boolean.class)); return this; } + /** + * Sets the bucket's website index page. + */ public Builder indexPage(String indexPage) { this.indexPage = indexPage; return this; } + /** + * Sets the bucket's website page to return when a resource is not found. + */ public Builder notFoundPage(String notFoundPage) { this.notFoundPage = notFoundPage; return this; } + /** + * Sets bucket's delete rules. + * + * @see Lifecycle Management + */ public Builder deleteRules(Iterable rules) { this.deleteRules = rules != null ? ImmutableList.copyOf(rules) : null; return this; } - public Builder storageClass(StorageClass storageClass) { - this.storageClass = firstNonNull(storageClass, StorageClass.NULL_VALUE); + /** + * Sets the bucket's storage class. A list of supported values is available + * here. + * + * @see Storage Classes + */ + public Builder storageClass(String storageClass) { + this.storageClass = storageClass; return this; } - public Builder location(Location location) { - this.location = firstNonNull(location, Location.NULL_VALUE); + /** + * Sets the bucket's location. A list of supported values is available + * here. + */ + public Builder location(String location) { + this.location = location; return this; } @@ -479,21 +430,43 @@ Builder metageneration(Long metageneration) { return this; } + /** + * Sets the bucket's Cross-Origin Resource Sharing (CORS) configuration. + * + * @see + * Cross-Origin Resource Sharing (CORS) + */ public Builder cors(Iterable cors) { this.cors = cors != null ? ImmutableList.copyOf(cors) : null; return this; } + /** + * Sets the bucket's access control configuration. + * + * @see + * About Access Control Lists + */ public Builder acl(Iterable acl) { this.acl = acl != null ? ImmutableList.copyOf(acl) : null; return this; } + /** + * Sets the default access control configuration to apply to bucket's blobs when no other + * configuration is specified. + * + * @see + * About Access Control Lists + */ public Builder defaultAcl(Iterable acl) { this.defaultAcl = acl != null ? ImmutableList.copyOf(acl) : null; return this; } + /** + * Creates a {@code BucketInfo} object. + */ public BucketInfo build() { checkNotNull(name); return new BucketInfo(this); @@ -519,70 +492,131 @@ private BucketInfo(Builder builder) { deleteRules = builder.deleteRules; } + /** + * Returns the bucket's id. + */ public String id() { return id; } + /** + * Returns the bucket's name. + */ public String name() { return name; } + /** + * Returns the bucket's owner. + */ public Entity owner() { return owner; } + /** + * Returns the URI of this bucket as a string. + */ public String selfLink() { return selfLink; } + /** + * Returns {@code true} if blob versioning is enabled for this bucket, {@code false} otherwise. + */ public Boolean versioningEnabled() { return Data.isNull(versioningEnabled) ? null : versioningEnabled; } + /** + * Returns bucket's website index page. + */ public String indexPage() { return indexPage; } + /** + * Returns bucket's website not found page, used we a resource could not be found. + */ public String notFoundPage() { return notFoundPage; } + /** + * Returns bucket's delete rules. + */ public List deleteRules() { return deleteRules; } + /** + * Returns bucket resource's entity tag. + */ public String etag() { return etag; } + /** + * Returns the time at which the bucket was created. + */ public Long createTime() { return createTime; } + /** + * Returns the metadata generation of this bucket. + */ public Long metageneration() { return metageneration; } - public Location location() { - return location == null || Data.isNull(location.value) ? null : location; + /** + * Returns the bucket's location. + * + * @see Bucket Locations + */ + public String location() { + return location; } - public StorageClass storageClass() { - return storageClass == null || Data.isNull(storageClass.value) ? null : storageClass; + /** + * Returns the bucket's storage class. + * + * @see Storage Classes + */ + public String storageClass() { + return storageClass; } + /** + * Returns the bucket's Cross-Origin Resource Sharing (CORS) configuration. + */ public List cors() { return cors; } + /** + * Returns the bucket's access control configuration. + * + * @see + * About Access Control Lists + */ public List acl() { return acl; } + /** + * Returns the default access control configuration for this bucket's blobs. + * + * @see + * About Access Control Lists + */ public List defaultAcl() { return defaultAcl; } + /** + * Returns a builder for the current bucket. + */ public Builder toBuilder() { return new Builder() .name(name) @@ -633,10 +667,10 @@ com.google.api.services.storage.model.Bucket toPb() { bucketPb.setMetageneration(metageneration); } if (location != null) { - bucketPb.setLocation(location.value()); + bucketPb.setLocation(location); } if (storageClass != null) { - bucketPb.setStorageClass(storageClass.value()); + bucketPb.setStorageClass(storageClass); } if (cors != null) { bucketPb.setCors(transform(cors, Cors.TO_PB_FUNCTION)); @@ -683,10 +717,16 @@ public Rule apply(DeleteRule deleteRule) { return bucketPb; } + /** + * Creates a {@code BucketInfo} object for the provided bucket name. + */ public static BucketInfo of(String name) { return builder(name).build(); } + /** + * Returns a {@code BucketInfo} builder where the bucket's name is set to the provided name. + */ public static Builder builder(String name) { return new Builder().name(name); } @@ -709,10 +749,10 @@ static BucketInfo fromPb(com.google.api.services.storage.model.Bucket bucketPb) builder.createTime(bucketPb.getTimeCreated().getValue()); } if (bucketPb.getLocation() != null) { - builder.location(Location.of(bucketPb.getLocation())); + builder.location(bucketPb.getLocation()); } if (bucketPb.getStorageClass() != null) { - builder.storageClass(StorageClass.of(bucketPb.getStorageClass())); + builder.storageClass(bucketPb.getStorageClass()); } if (bucketPb.getCors() != null) { builder.cors(transform(bucketPb.getCors(), Cors.FROM_PB_FUNCTION)); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java index 09ba0e8cda8e..4fa420b4b6e1 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java @@ -31,10 +31,8 @@ import com.google.gcloud.storage.BucketInfo.DeleteRule; import com.google.gcloud.storage.BucketInfo.DeleteRule.Type; import com.google.gcloud.storage.BucketInfo.IsLiveDeleteRule; -import com.google.gcloud.storage.BucketInfo.Location; import com.google.gcloud.storage.BucketInfo.NumNewerVersionsDeleteRule; import com.google.gcloud.storage.BucketInfo.RawDeleteRule; -import com.google.gcloud.storage.BucketInfo.StorageClass; import org.junit.Test; @@ -59,8 +57,8 @@ public class BucketInfoTest { Collections.singletonList(new AgeDeleteRule(5)); private static final String INDEX_PAGE = "index.html"; private static final String NOT_FOUND_PAGE = "error.html"; - private static final Location LOCATION = Location.asia(); - private static final StorageClass STORAGE_CLASS = StorageClass.standard(); + private static final String LOCATION = "ASIA"; + private static final String STORAGE_CLASS = "STANDARD"; private static final Boolean VERSIONING_ENABLED = true; private static final BucketInfo BUCKET_INFO = BucketInfo.builder("b") .acl(ACL) @@ -149,16 +147,6 @@ private void compareBuckets(BucketInfo expected, BucketInfo value) { assertEquals(expected.versioningEnabled(), value.versioningEnabled()); } - @Test - public void testLocation() { - assertEquals("ASIA", Location.asia().value()); - assertEquals("EU", Location.eu().value()); - assertEquals("US", Location.us().value()); - assertSame(Location.asia(), Location.of("asia")); - assertSame(Location.eu(), Location.of("EU")); - assertSame(Location.us(), Location.of("uS")); - } - @Test public void testDeleteRules() { AgeDeleteRule ageRule = new AgeDeleteRule(10); From ecaf7519522b43431f4c7c24b31b0bd4a09c54ed Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 17:14:52 +0100 Subject: [PATCH 25/94] Add javadoc to BlobInfo --- .../com/google/gcloud/storage/BlobInfo.java | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java index 01711a53613e..9662635160e8 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java @@ -123,6 +123,9 @@ public static final class Builder { private Builder() {} + /** + * Sets the blob identity. + */ public Builder blobId(BlobId blobId) { this.blobId = checkNotNull(blobId); return this; @@ -133,21 +136,33 @@ Builder id(String id) { return this; } + /** + * Sets the blob's data content type. + */ public Builder contentType(String contentType) { this.contentType = firstNonNull(contentType, Data.nullOf(String.class)); return this; } + /** + * Sets the blob's data content disposition. + */ public Builder contentDisposition(String contentDisposition) { this.contentDisposition = firstNonNull(contentDisposition, Data.nullOf(String.class)); return this; } + /** + * Sets the blob's data content language. + */ public Builder contentLanguage(String contentLanguage) { this.contentLanguage = firstNonNull(contentLanguage, Data.nullOf(String.class)); return this; } + /** + * Sets the blob's data content encoding. + */ public Builder contentEncoding(String contentEncoding) { this.contentEncoding = firstNonNull(contentEncoding, Data.nullOf(String.class)); return this; @@ -158,11 +173,20 @@ Builder componentCount(Integer componentCount) { return this; } + /** + * Sets the blob's data cache control. + */ public Builder cacheControl(String cacheControl) { this.cacheControl = firstNonNull(cacheControl, Data.nullOf(String.class)); return this; } + /** + * Sets the blob's access control configuration. + * + * @see + * About Access Control Lists + */ public Builder acl(List acl) { this.acl = acl != null ? ImmutableList.copyOf(acl) : null; return this; @@ -188,11 +212,18 @@ Builder selfLink(String selfLink) { return this; } + /** + * Sets the MD5 hash of blob's data. MD5 value must be encoded in base64. + */ public Builder md5(String md5) { this.md5 = firstNonNull(md5, Data.nullOf(String.class)); return this; } + /** + * Sets the CRC32C checksum of blob's data. CRC32C value must be encoded in base64 in big-endian + * order. + */ public Builder crc32c(String crc32c) { this.crc32c = firstNonNull(crc32c, Data.nullOf(String.class)); return this; @@ -203,6 +234,9 @@ Builder mediaLink(String mediaLink) { return this; } + /** + * Sets the blob's metadata. + */ public Builder metadata(Map metadata) { this.metadata = metadata != null ? new HashMap(metadata) : Data.nullOf(ImmutableEmptyMap.class); @@ -229,6 +263,9 @@ Builder updateTime(Long updateTime) { return this; } + /** + * Creates a {@code BlobInfo} object. + */ public BlobInfo build() { checkNotNull(blobId); return new BlobInfo(this); @@ -259,98 +296,177 @@ private BlobInfo(Builder builder) { updateTime = builder.updateTime; } + /** + * Returns the blob's identity. + */ public BlobId blobId() { return blobId; } + /** + * Returns the name of the containing bucket. + */ public String bucket() { return blobId().bucket(); } + /** + * Returns the blob's id. + */ public String id() { return id; } + /** + * Returns the blob's name. + */ public String name() { return blobId().name(); } + /** + * Returns the blob's data cache control. + */ public String cacheControl() { return Data.isNull(cacheControl) ? null : cacheControl; } + /** + * Returns the blob's access control configuration. + * + * @see + * About Access Control Lists + */ public List acl() { return acl; } + /** + * Returns the blob's owner. + */ public Acl.Entity owner() { return owner; } + /** + * Returns the blob's data size in bytes. + */ public Long size() { return size; } + /** + * Returns the blob's data content type. + */ public String contentType() { return Data.isNull(contentType) ? null : contentType; } + /** + * Returns the blob's data content encoding. + */ public String contentEncoding() { return Data.isNull(contentEncoding) ? null : contentEncoding; } + /** + * Returns the blob's data content disposition. + */ public String contentDisposition() { return Data.isNull(contentDisposition) ? null : contentDisposition; } + /** + * Returns the blob's data content language. + */ public String contentLanguage() { return Data.isNull(contentLanguage) ? null : contentLanguage; } + /** + * Returns the number of components that make up this object. Components are accumulated through + * the {@link Storage#compose(Storage.ComposeRequest)} operation. + * + * @see Component Count + * Property + */ public Integer componentCount() { return componentCount; } + /** + * Returns blob resource's entity tag. + */ public String etag() { return etag; } + /** + * Returns the URI of this blob as a string. + */ public String selfLink() { return selfLink; } + /** + * Returns the MD5 hash of blob's data encoded in base64. + */ public String md5() { return Data.isNull(md5) ? null : md5; } + /** + * Returns the CRC32C checksum of blob's data encoded in base64 in big-endian order. + */ public String crc32c() { return Data.isNull(crc32c) ? null : crc32c; } + /** + * Returns the blob's media download link. + */ public String mediaLink() { return mediaLink; } + /** + * Returns blob's metadata. + */ public Map metadata() { return metadata == null || Data.isNull(metadata) ? null : Collections.unmodifiableMap(metadata); } + /** + * Returns blob's data generation. + */ public Long generation() { return generation; } + /** + * Returns blob's metageneration. + */ public Long metageneration() { return metageneration; } + /** + * Returns the deletion time of the blob. + */ public Long deleteTime() { return deleteTime; } + /** + * Returns the last modification time of the blob's metadata. + */ public Long updateTime() { return updateTime; } + /** + * Returns a builder for the current blob. + */ public Builder toBuilder() { return new Builder() .blobId(blobId) @@ -444,14 +560,23 @@ public ObjectAccessControl apply(Acl acl) { return storageObject; } + /** + * Returns a {@code BlobInfo} builder where blob identity is set using the provided values. + */ public static Builder builder(BucketInfo bucketInfo, String name) { return builder(bucketInfo.name(), name); } + /** + * Returns a {@code BlobInfo} builder where blob identity is set using the provided values. + */ public static Builder builder(String bucket, String name) { return new Builder().blobId(BlobId.of(bucket, name)); } + /** + * Returns a {@code BlobInfo} builder where blob identity is set to the provided value. + */ public static Builder builder(BlobId blobId) { return new Builder().blobId(blobId); } From b9ef50883ddda7e19f1f9ec4599dca4c279c6ce8 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 17:15:08 +0100 Subject: [PATCH 26/94] Add javadoc to Storage --- .../com/google/gcloud/storage/Storage.java | 222 +++++++++++++++++- 1 file changed, 221 insertions(+), 1 deletion(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 115449064c66..128099b7eab3 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -156,6 +156,9 @@ static String selector(BlobField... fields) { } } + /** + * Class for specifying bucket target options. + */ class BucketTargetOption extends Option { private static final long serialVersionUID = -5880204616982900975L; @@ -168,23 +171,40 @@ private BucketTargetOption(StorageRpc.Option rpcOption) { this(rpcOption, null); } + /** + * Returns an option for specifying bucket's predefined ACL configuration. + */ public static BucketTargetOption predefinedAcl(PredefinedAcl acl) { return new BucketTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry()); } + /** + * Returns an option for specifying bucket's default ACL configuration for blobs. + */ public static BucketTargetOption predefinedDefaultObjectAcl(PredefinedAcl acl) { return new BucketTargetOption(StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL, acl.entry()); } + /** + * Returns an option for bucket's metageneration match. If this option is used the request will + * fail if metageneration does not match. + */ public static BucketTargetOption metagenerationMatch() { return new BucketTargetOption(StorageRpc.Option.IF_METAGENERATION_MATCH); } + /** + * Returns an option for bucket's metageneration mismatch. If this option is used the request + * will fail if metageneration matches. + */ public static BucketTargetOption metagenerationNotMatch() { return new BucketTargetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH); } } + /** + * Class for specifying bucket source options. + */ class BucketSourceOption extends Option { private static final long serialVersionUID = 5185657617120212117L; @@ -193,15 +213,26 @@ private BucketSourceOption(StorageRpc.Option rpcOption, long metageneration) { super(rpcOption, metageneration); } + /** + * Returns an option for bucket's metageneration match. If this option is used the request will + * fail if bucket's metageneration does not match the provided value. + */ public static BucketSourceOption metagenerationMatch(long metageneration) { return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH, metageneration); } + /** + * Returns an option for bucket's metageneration mismatch. If this option is used the request + * will fail if bucket's metageneration matches the provided value. + */ public static BucketSourceOption metagenerationNotMatch(long metageneration) { return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration); } } + /** + * Class for specifying bucket source options. + */ class BucketGetOption extends Option { private static final long serialVersionUID = 1901844869484087395L; @@ -214,10 +245,18 @@ private BucketGetOption(StorageRpc.Option rpcOption, String value) { super(rpcOption, value); } + /** + * Returns an option for bucket's metageneration match. If this option is used the request will + * fail if bucket's metageneration does not match the provided value. + */ public static BucketGetOption metagenerationMatch(long metageneration) { return new BucketGetOption(StorageRpc.Option.IF_METAGENERATION_MATCH, metageneration); } + /** + * Returns an option for bucket's metageneration mismatch. If this option is used the request + * will fail if bucket's metageneration matches the provided value. + */ public static BucketGetOption metagenerationNotMatch(long metageneration) { return new BucketGetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration); } @@ -233,6 +272,9 @@ public static BucketGetOption fields(BucketField... fields) { } } + /** + * Class for specifying blob target options. + */ class BlobTargetOption extends Option { private static final long serialVersionUID = 214616862061934846L; @@ -245,26 +287,48 @@ private BlobTargetOption(StorageRpc.Option rpcOption) { this(rpcOption, null); } + /** + * Returns an option for specifying blob's predefined ACL configuration. + */ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) { return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry()); } + /** + * Returns an option that causes an operation to succeed only if the target blob does not exist. + */ public static BlobTargetOption doesNotExist() { return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0L); } + /** + * Returns an option for blob's data generation match. If this option is used the request will + * fail if generation does not match. + */ public static BlobTargetOption generationMatch() { return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH); } + /** + * Returns an option for blob's data generation mismatch. If this option is used the request + * will fail if generation matches. + */ public static BlobTargetOption generationNotMatch() { return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH); } + /** + * Returns an option for blob's metageneration match. If this option is used the request will + * fail if metageneration does not match. + */ public static BlobTargetOption metagenerationMatch() { return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_MATCH); } + /** + * Returns an option for blob's metageneration mismatch. If this option is used the request will + * fail if metageneration matches. + */ public static BlobTargetOption metagenerationNotMatch() { return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH); } @@ -290,6 +354,9 @@ static Tuple convert(BlobInfo info, BlobWriteOptio } } + /** + * Class for specifying blob write options. + */ class BlobWriteOption implements Serializable { private static final long serialVersionUID = -3880421670966224580L; @@ -336,39 +403,72 @@ public boolean equals(Object obj) { return this.option == other.option && Objects.equals(this.value, other.value); } + /** + * Returns an option for specifying blob's predefined ACL configuration. + */ public static BlobWriteOption predefinedAcl(PredefinedAcl acl) { return new BlobWriteOption(Option.PREDEFINED_ACL, acl.entry()); } + /** + * Returns an option that causes an operation to succeed only if the target blob does not exist. + */ public static BlobWriteOption doesNotExist() { return new BlobWriteOption(Option.IF_GENERATION_MATCH, 0L); } + /** + * Returns an option for blob's data generation match. If this option is used the request will + * fail if generation does not match. + */ public static BlobWriteOption generationMatch() { return new BlobWriteOption(Option.IF_GENERATION_MATCH); } + /** + * Returns an option for blob's data generation mismatch. If this option is used the request + * will fail if generation matches. + */ public static BlobWriteOption generationNotMatch() { return new BlobWriteOption(Option.IF_GENERATION_NOT_MATCH); } + /** + * Returns an option for blob's metageneration match. If this option is used the request will + * fail if metageneration does not match. + */ public static BlobWriteOption metagenerationMatch() { return new BlobWriteOption(Option.IF_METAGENERATION_MATCH); } + /** + * Returns an option for blob's metageneration mismatch. If this option is used the request will + * fail if metageneration matches. + */ public static BlobWriteOption metagenerationNotMatch() { return new BlobWriteOption(Option.IF_METAGENERATION_NOT_MATCH); } + /** + * Returns an option for blob's data MD5 hash match. If this option is used the request will + * fail if blobs' data MD5 hash does not match. + */ public static BlobWriteOption md5Match() { return new BlobWriteOption(Option.IF_MD5_MATCH, true); } + /** + * Returns an option for blob's data CRC32C checksum match. If this option is used the request + * will fail if blobs' data CRC32C checksum does not match. + */ public static BlobWriteOption crc32cMatch() { return new BlobWriteOption(Option.IF_CRC32C_MATCH, true); } } + /** + * Class for specifying blob source options. + */ class BlobSourceOption extends Option { private static final long serialVersionUID = -3712768261070182991L; @@ -377,23 +477,42 @@ private BlobSourceOption(StorageRpc.Option rpcOption, long value) { super(rpcOption, value); } + /** + * Returns an option for blob's data generation match. If this option is used the request will + * fail if blob's generation does not match the provided value. + */ public static BlobSourceOption generationMatch(long generation) { return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_MATCH, generation); } + /** + * Returns an option for blob's data generation mismatch. If this option is used the request + * will fail if blob's generation matches the provided value. + */ public static BlobSourceOption generationNotMatch(long generation) { return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH, generation); } + /** + * Returns an option for blob's metageneration match. If this option is used the request will + * fail if blob's metageneration does not match the provided value. + */ public static BlobSourceOption metagenerationMatch(long metageneration) { return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH, metageneration); } + /** + * Returns an option for blob's metageneration mismatch. If this option is used the request will + * fail if blob's metageneration matches the provided value. + */ public static BlobSourceOption metagenerationNotMatch(long metageneration) { return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration); } } + /** + * Class for specifying blob get options. + */ class BlobGetOption extends Option { private static final long serialVersionUID = 803817709703661480L; @@ -406,18 +525,34 @@ private BlobGetOption(StorageRpc.Option rpcOption, String value) { super(rpcOption, value); } + /** + * Returns an option for blob's data generation match. If this option is used the request will + * fail if blob's generation does not match the provided value. + */ public static BlobGetOption generationMatch(long generation) { return new BlobGetOption(StorageRpc.Option.IF_GENERATION_MATCH, generation); } + /** + * Returns an option for blob's data generation mismatch. If this option is used the request + * will fail if blob's generation matches the provided value. + */ public static BlobGetOption generationNotMatch(long generation) { return new BlobGetOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH, generation); } + /** + * Returns an option for blob's metageneration match. If this option is used the request will + * fail if blob's metageneration does not match the provided value. + */ public static BlobGetOption metagenerationMatch(long metageneration) { return new BlobGetOption(StorageRpc.Option.IF_METAGENERATION_MATCH, metageneration); } + /** + * Returns an option for blob's metageneration mismatch. If this option is used the request will + * fail if blob's metageneration matches the provided value. + */ public static BlobGetOption metagenerationNotMatch(long metageneration) { return new BlobGetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration); } @@ -433,6 +568,9 @@ public static BlobGetOption fields(BlobField... fields) { } } + /** + * Class for specifying bucket list options. + */ class BucketListOption extends Option { private static final long serialVersionUID = 8754017079673290353L; @@ -441,14 +579,23 @@ private BucketListOption(StorageRpc.Option option, Object value) { super(option, value); } + /** + * Returns an option to specify the maximum number of buckets to be returned. + */ public static BucketListOption maxResults(long maxResults) { return new BucketListOption(StorageRpc.Option.MAX_RESULTS, maxResults); } + /** + * Returns an option to specify the page token from which to start listing buckets. + */ public static BucketListOption startPageToken(String pageToken) { return new BucketListOption(StorageRpc.Option.PAGE_TOKEN, pageToken); } + /** + * Returns an option to set a prefix for listed buckets names. + */ public static BucketListOption prefix(String prefix) { return new BucketListOption(StorageRpc.Option.PREFIX, prefix); } @@ -466,6 +613,9 @@ public static BucketListOption fields(BucketField... fields) { } } + /** + * Class for specifying blob list options. + */ class BlobListOption extends Option { private static final long serialVersionUID = 9083383524788661294L; @@ -474,18 +624,30 @@ private BlobListOption(StorageRpc.Option option, Object value) { super(option, value); } + /** + * Returns an option to specify the maximum number of blobs to be returned. + */ public static BlobListOption maxResults(long maxResults) { return new BlobListOption(StorageRpc.Option.MAX_RESULTS, maxResults); } + /** + * Returns an option to specify the page token from which to start listing blobs. + */ public static BlobListOption startPageToken(String pageToken) { return new BlobListOption(StorageRpc.Option.PAGE_TOKEN, pageToken); } + /** + * Returns an option to set a prefix for listed blobs names. + */ public static BlobListOption prefix(String prefix) { return new BlobListOption(StorageRpc.Option.PREFIX, prefix); } + /** + * Returns an option to specify whether blob listing should include subdirectories or not. + */ public static BlobListOption recursive(boolean recursive) { return new BlobListOption(StorageRpc.Option.DELIMITER, recursive); } @@ -503,6 +665,9 @@ public static BlobListOption fields(BlobField... fields) { } } + /** + * Class for specifying signed URL options. + */ class SignUrlOption implements Serializable { private static final long serialVersionUID = 7850569877451099267L; @@ -562,6 +727,12 @@ public static SignUrlOption serviceAccount(ServiceAccountAuthCredentials credent } } + /** + * A class to contain all information needed for a Google Cloud Storage Compose operation. + * + * @see + * Compose Operation + */ class ComposeRequest implements Serializable { private static final long serialVersionUID = -7385681353748590911L; @@ -570,6 +741,9 @@ class ComposeRequest implements Serializable { private final BlobInfo target; private final List targetOptions; + /** + * Class for Compose source blobs. + */ public static class SourceBlob implements Serializable { private static final long serialVersionUID = 4094962795951990439L; @@ -601,6 +775,9 @@ public static class Builder { private final Set targetOptions = new LinkedHashSet<>(); private BlobInfo target; + /** + * Add source blobs for compose operation. + */ public Builder addSource(Iterable blobs) { for (String blob : blobs) { sourceBlobs.add(new SourceBlob(blob)); @@ -608,6 +785,9 @@ public Builder addSource(Iterable blobs) { return this; } + /** + * Add source blobs for compose operation. + */ public Builder addSource(String... blobs) { return addSource(Arrays.asList(blobs)); } @@ -620,21 +800,33 @@ public Builder addSource(String blob, long generation) { return this; } + /** + * Sets compose operation's target blob. + */ public Builder target(BlobInfo target) { this.target = target; return this; } + /** + * Sets compose operation's target blob options. + */ public Builder targetOptions(BlobTargetOption... options) { Collections.addAll(targetOptions, options); return this; } + /** + * Sets compose operation's target blob options. + */ public Builder targetOptions(Iterable options) { Iterables.addAll(targetOptions, options); return this; } + /** + * Creates a {@code ComposeRequest} object. + */ public ComposeRequest build() { checkArgument(!sourceBlobs.isEmpty()); checkNotNull(target); @@ -648,31 +840,59 @@ private ComposeRequest(Builder builder) { targetOptions = ImmutableList.copyOf(builder.targetOptions); } + /** + * Returns compose operation's source blobs. + */ public List sourceBlobs() { return sourceBlobs; } + /** + * Returns compose operation's target blob. + */ public BlobInfo target() { return target; } + /** + * Returns compose operation's target blob's options. + */ public List targetOptions() { return targetOptions; } + /** + * Creates a {@code ComposeRequest} object. + * + * @param sources source blobs names + * @param target target blob + */ public static ComposeRequest of(Iterable sources, BlobInfo target) { return builder().target(target).addSource(sources).build(); } + /** + * Creates a {@code ComposeRequest} object. + * + * @param bucket name of the bucket where the compose operation takes place + * @param sources source blobs names + * @param target target blob name + */ public static ComposeRequest of(String bucket, Iterable sources, String target) { return of(sources, BlobInfo.builder(BlobId.of(bucket, target)).build()); } + /** + * Returns a {@code ComposeRequest} builder. + */ public static Builder builder() { return new Builder(); } } + /** + * A class to contain all information needed for a Google Cloud Storage Copy operation. + */ class CopyRequest implements Serializable { private static final long serialVersionUID = -4498650529476219937L; @@ -786,7 +1006,7 @@ public Builder megabytesCopiedPerChunk(Long megabytesCopiedPerChunk) { } /** - * Creates a {@code CopyRequest}. + * Creates a {@code CopyRequest} object. */ public CopyRequest build() { checkNotNull(source); From 3125167ad845cc5f0c5deeeffa215a56e9fff896 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 17:55:34 +0100 Subject: [PATCH 27/94] Add getDefaultInstance method to StorageOptions and DatastoreOptions --- .../java/com/google/gcloud/datastore/DatastoreOptions.java | 7 +++++++ .../java/com/google/gcloud/storage/StorageOptions.java | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java index 5338c03a6d56..58c361229f07 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java @@ -171,6 +171,13 @@ public String namespace() { return namespace; } + /** + * Returns a default {@code DatastoreOptions} instance. + */ + public static DatastoreOptions getDefaultInstance() { + return builder().build(); + } + private static String defaultNamespace() { try { Class clazz = Class.forName("com.google.appengine.api.NamespaceManager"); diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java index 16c17c3e8d98..4b295fc97b29 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java @@ -111,6 +111,13 @@ public String pathDelimiter() { return pathDelimiter; } + /** + * Returns a default {@code StorageOptions} instance. + */ + public static StorageOptions getDefaultInstance() { + return builder().build(); + } + @Override public Builder toBuilder() { return new Builder(this); From d7845b86bcc38abc5a444fd8f08ec98a0d73d0c5 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 21:39:44 +0100 Subject: [PATCH 28/94] Rename getDefaultInstance to defaultInstance and update READMEs --- README.md | 2 +- gcloud-java-datastore/README.md | 2 +- .../main/java/com/google/gcloud/datastore/DatastoreOptions.java | 2 +- .../src/main/java/com/google/gcloud/storage/StorageOptions.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0fd576a6a127..5ed44e91adfd 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ import com.google.gcloud.datastore.Entity; import com.google.gcloud.datastore.Key; import com.google.gcloud.datastore.KeyFactory; -Datastore datastore = DatastoreOptions.getDefaultInstance().service(); +Datastore datastore = DatastoreOptions.defaultInstance().service(); KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND); Key key = keyFactory.newKey(keyName); Entity entity = datastore.get(key); diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md index 67483fae2b6e..6d9fc0e8c4d6 100644 --- a/gcloud-java-datastore/README.md +++ b/gcloud-java-datastore/README.md @@ -66,7 +66,7 @@ import com.google.gcloud.datastore.Entity; import com.google.gcloud.datastore.Key; import com.google.gcloud.datastore.KeyFactory; -Datastore datastore = DatastoreOptions.getDefaultInstance().service(); +Datastore datastore = DatastoreOptions.defaultInstance().service(); KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND); Key key = keyFactory.newKey(keyName); Entity entity = datastore.get(key); diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java index 58c361229f07..ee15393a8048 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java @@ -174,7 +174,7 @@ public String namespace() { /** * Returns a default {@code DatastoreOptions} instance. */ - public static DatastoreOptions getDefaultInstance() { + public static DatastoreOptions defaultInstance() { return builder().build(); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java index 4b295fc97b29..2ad0950aa6aa 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java @@ -114,7 +114,7 @@ public String pathDelimiter() { /** * Returns a default {@code StorageOptions} instance. */ - public static StorageOptions getDefaultInstance() { + public static StorageOptions defaultInstance() { return builder().build(); } From d0265406392a9c30f96903c5647874e82f92117b Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 21:50:51 +0100 Subject: [PATCH 29/94] Make RetryParams conform with our code style --- .../java/com/google/gcloud/RetryHelper.java | 14 ++++---- .../java/com/google/gcloud/RetryParams.java | 30 ++++++++-------- .../com/google/gcloud/RetryHelperTest.java | 2 +- .../com/google/gcloud/RetryParamsTest.java | 34 +++++++++---------- .../gcloud/datastore/DatastoreTest.java | 6 ++-- .../gcloud/datastore/SerializationTest.java | 2 +- .../gcloud/examples/StorageExample.java | 3 +- .../gcloud/storage/RemoteGcsHelperTest.java | 10 +++--- .../gcloud/storage/SerializationTest.java | 6 ++-- .../gcloud/storage/StorageImplTest.java | 6 ++-- 10 files changed, 56 insertions(+), 57 deletions(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/RetryHelper.java b/gcloud-java-core/src/main/java/com/google/gcloud/RetryHelper.java index 7b47209cd3ff..90f403758e45 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/RetryHelper.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/RetryHelper.java @@ -194,9 +194,9 @@ private V doRetry() throws RetryHelperException { } exception = e; } - if (attemptNumber >= params.getRetryMaxAttempts() - || attemptNumber >= params.getRetryMinAttempts() - && stopwatch.elapsed(MILLISECONDS) >= params.getTotalRetryPeriodMillis()) { + if (attemptNumber >= params.retryMaxAttempts() + || attemptNumber >= params.retryMinAttempts() + && stopwatch.elapsed(MILLISECONDS) >= params.totalRetryPeriodMillis()) { throw new RetriesExhaustedException(this + ": Too many failures, giving up", exception); } long sleepDurationMillis = getSleepDuration(params, attemptNumber); @@ -215,9 +215,9 @@ private V doRetry() throws RetryHelperException { @VisibleForTesting static long getSleepDuration(RetryParams retryParams, int attemptsSoFar) { - long initialDelay = retryParams.getInitialRetryDelayMillis(); - double backoffFactor = retryParams.getRetryDelayBackoffFactor(); - long maxDelay = retryParams.getMaxRetryDelayMillis(); + long initialDelay = retryParams.initialRetryDelayMillis(); + double backoffFactor = retryParams.retryDelayBackoffFactor(); + long maxDelay = retryParams.maxRetryDelayMillis(); long retryDelay = getExponentialValue(initialDelay, backoffFactor, maxDelay, attemptsSoFar); return (long) ((random() / 2.0 + .75) * retryDelay); } @@ -228,7 +228,7 @@ private static long getExponentialValue(long initialDelay, double backoffFactor, } public static V runWithRetries(Callable callable) throws RetryHelperException { - return runWithRetries(callable, RetryParams.getDefaultInstance(), + return runWithRetries(callable, RetryParams.defaultInstance(), ExceptionHandler.getDefaultInstance()); } diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/RetryParams.java b/gcloud-java-core/src/main/java/com/google/gcloud/RetryParams.java index 461dbac77ff2..ab3644c6d747 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/RetryParams.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/RetryParams.java @@ -38,8 +38,8 @@ * {@code RetryParams}, first create a {@link RetryParams.Builder}. The builder is mutable and each * of the parameters can be set (any unset parameters will fallback to the defaults). The * {@code Builder} can be then used to create an immutable {@code RetryParams} object. For default - * {@code RetryParams} use {@link #getDefaultInstance}. Default settings are subject to change - * release to release. If you require specific settings, explicitly create an instance of + * {@code RetryParams} use {@link #defaultInstance}. Default settings are subject to change release + * to release. If you require specific settings, explicitly create an instance of * {@code RetryParams} with all the required settings. * * @see RetryHelper @@ -91,12 +91,12 @@ private Builder() { retryDelayBackoffFactor = DEFAULT_RETRY_DELAY_BACKOFF_FACTOR; totalRetryPeriodMillis = DEFAULT_TOTAL_RETRY_PERIOD_MILLIS; } else { - retryMinAttempts = retryParams.getRetryMinAttempts(); - retryMaxAttempts = retryParams.getRetryMaxAttempts(); - initialRetryDelayMillis = retryParams.getInitialRetryDelayMillis(); - maxRetryDelayMillis = retryParams.getMaxRetryDelayMillis(); - retryDelayBackoffFactor = retryParams.getRetryDelayBackoffFactor(); - totalRetryPeriodMillis = retryParams.getTotalRetryPeriodMillis(); + retryMinAttempts = retryParams.retryMinAttempts(); + retryMaxAttempts = retryParams.retryMaxAttempts(); + initialRetryDelayMillis = retryParams.initialRetryDelayMillis(); + maxRetryDelayMillis = retryParams.maxRetryDelayMillis(); + retryDelayBackoffFactor = retryParams.retryDelayBackoffFactor(); + totalRetryPeriodMillis = retryParams.totalRetryPeriodMillis(); } } @@ -201,7 +201,7 @@ private RetryParams(Builder builder) { /** * Returns an instance with the default parameters. */ - public static RetryParams getDefaultInstance() { + public static RetryParams defaultInstance() { return DEFAULT_INSTANCE; } @@ -216,14 +216,14 @@ public static RetryParams noRetries() { /** * Returns the retryMinAttempts. Default value is {@value #DEFAULT_RETRY_MIN_ATTEMPTS}. */ - public int getRetryMinAttempts() { + public int retryMinAttempts() { return retryMinAttempts; } /** * Returns the retryMaxAttempts. Default value is {@value #DEFAULT_RETRY_MAX_ATTEMPTS}. */ - public int getRetryMaxAttempts() { + public int retryMaxAttempts() { return retryMaxAttempts; } @@ -231,14 +231,14 @@ public int getRetryMaxAttempts() { * Returns the initialRetryDelayMillis. Default value is * {@value #DEFAULT_INITIAL_RETRY_DELAY_MILLIS}. */ - public long getInitialRetryDelayMillis() { + public long initialRetryDelayMillis() { return initialRetryDelayMillis; } /** * Returns the maxRetryDelayMillis. Default values is {@value #DEFAULT_MAX_RETRY_DELAY_MILLIS}. */ - public long getMaxRetryDelayMillis() { + public long maxRetryDelayMillis() { return maxRetryDelayMillis; } @@ -246,7 +246,7 @@ public long getMaxRetryDelayMillis() { * Returns the maxRetryDelayBackoffFactor. Default values is * {@value #DEFAULT_RETRY_DELAY_BACKOFF_FACTOR}. */ - public double getRetryDelayBackoffFactor() { + public double retryDelayBackoffFactor() { return retryDelayBackoffFactor; } @@ -254,7 +254,7 @@ public double getRetryDelayBackoffFactor() { * Returns the totalRetryPeriodMillis. Default value is * {@value #DEFAULT_TOTAL_RETRY_PERIOD_MILLIS}. */ - public long getTotalRetryPeriodMillis() { + public long totalRetryPeriodMillis() { return totalRetryPeriodMillis; } diff --git a/gcloud-java-core/src/test/java/com/google/gcloud/RetryHelperTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/RetryHelperTest.java index dfd933bcae46..bf11e744dbbf 100644 --- a/gcloud-java-core/src/test/java/com/google/gcloud/RetryHelperTest.java +++ b/gcloud-java-core/src/test/java/com/google/gcloud/RetryHelperTest.java @@ -118,7 +118,7 @@ public void testTriesAtLeastMinTimes() { @Override public Integer call() throws IOException { timesCalled++; assertEquals(timesCalled, RetryHelper.getContext().getAttemptNumber()); - assertEquals(10, RetryHelper.getContext().getRetryParams().getRetryMaxAttempts()); + assertEquals(10, RetryHelper.getContext().getRetryParams().retryMaxAttempts()); if (timesCalled <= timesToFail) { throw new IOException(); } diff --git a/gcloud-java-core/src/test/java/com/google/gcloud/RetryParamsTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/RetryParamsTest.java index d1d5e3c076d8..eae44693929b 100644 --- a/gcloud-java-core/src/test/java/com/google/gcloud/RetryParamsTest.java +++ b/gcloud-java-core/src/test/java/com/google/gcloud/RetryParamsTest.java @@ -41,15 +41,15 @@ public class RetryParamsTest { @Test public void testDefaults() { - RetryParams params1 = RetryParams.getDefaultInstance(); + RetryParams params1 = RetryParams.defaultInstance(); RetryParams params2 = RetryParams.builder().build(); for (RetryParams params : Arrays.asList(params1, params2)) { - assertEquals(DEFAULT_INITIAL_RETRY_DELAY_MILLIS, params.getInitialRetryDelayMillis()); - assertEquals(DEFAULT_MAX_RETRY_DELAY_MILLIS, params.getMaxRetryDelayMillis()); - assertEquals(DEFAULT_RETRY_DELAY_BACKOFF_FACTOR, params.getRetryDelayBackoffFactor(), 0); - assertEquals(DEFAULT_RETRY_MAX_ATTEMPTS, params.getRetryMaxAttempts()); - assertEquals(DEFAULT_RETRY_MIN_ATTEMPTS, params.getRetryMinAttempts()); - assertEquals(DEFAULT_TOTAL_RETRY_PERIOD_MILLIS, params.getTotalRetryPeriodMillis()); + assertEquals(DEFAULT_INITIAL_RETRY_DELAY_MILLIS, params.initialRetryDelayMillis()); + assertEquals(DEFAULT_MAX_RETRY_DELAY_MILLIS, params.maxRetryDelayMillis()); + assertEquals(DEFAULT_RETRY_DELAY_BACKOFF_FACTOR, params.retryDelayBackoffFactor(), 0); + assertEquals(DEFAULT_RETRY_MAX_ATTEMPTS, params.retryMaxAttempts()); + assertEquals(DEFAULT_RETRY_MIN_ATTEMPTS, params.retryMinAttempts()); + assertEquals(DEFAULT_TOTAL_RETRY_PERIOD_MILLIS, params.totalRetryPeriodMillis()); } } @@ -65,12 +65,12 @@ public void testSetAndCopy() { RetryParams params1 = builder.build(); RetryParams params2 = new RetryParams.Builder(params1).build(); for (RetryParams params : Arrays.asList(params1, params2)) { - assertEquals(101, params.getInitialRetryDelayMillis()); - assertEquals(102, params.getMaxRetryDelayMillis()); - assertEquals(103, params.getRetryDelayBackoffFactor(), 0); - assertEquals(107, params.getRetryMinAttempts()); - assertEquals(108, params.getRetryMaxAttempts()); - assertEquals(109, params.getTotalRetryPeriodMillis()); + assertEquals(101, params.initialRetryDelayMillis()); + assertEquals(102, params.maxRetryDelayMillis()); + assertEquals(103, params.retryDelayBackoffFactor(), 0); + assertEquals(107, params.retryMinAttempts()); + assertEquals(108, params.retryMaxAttempts()); + assertEquals(109, params.totalRetryPeriodMillis()); } } @@ -79,19 +79,19 @@ public void testBadSettings() { RetryParams.Builder builder = RetryParams.builder(); builder.initialRetryDelayMillis(-1); builder = assertFailure(builder); - builder.maxRetryDelayMillis(RetryParams.getDefaultInstance().getInitialRetryDelayMillis() - 1); + builder.maxRetryDelayMillis(RetryParams.defaultInstance().initialRetryDelayMillis() - 1); builder = assertFailure(builder); builder.retryDelayBackoffFactor(-1); builder = assertFailure(builder); builder.retryMinAttempts(-1); builder = assertFailure(builder); - builder.retryMaxAttempts(RetryParams.getDefaultInstance().getRetryMinAttempts() - 1); + builder.retryMaxAttempts(RetryParams.defaultInstance().retryMinAttempts() - 1); builder = assertFailure(builder); builder.totalRetryPeriodMillis(-1); builder = assertFailure(builder); // verify that it is OK for min and max to be equal - builder.retryMaxAttempts(RetryParams.getDefaultInstance().getRetryMinAttempts()); - builder.maxRetryDelayMillis(RetryParams.getDefaultInstance().getInitialRetryDelayMillis()); + builder.retryMaxAttempts(RetryParams.defaultInstance().retryMinAttempts()); + builder.maxRetryDelayMillis(RetryParams.defaultInstance().initialRetryDelayMillis()); builder.build(); } diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java index 6afd40927264..162104c0dd84 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java @@ -625,7 +625,7 @@ private Datastore createDatastoreForDeferredLookup() throws DatastoreRpcExceptio EasyMock.replay(rpcFactoryMock, rpcMock); DatastoreOptions options = this.options.toBuilder() - .retryParams(RetryParams.getDefaultInstance()) + .retryParams(RetryParams.defaultInstance()) .serviceRpcFactory(rpcFactoryMock) .build(); return options.service(); @@ -738,7 +738,7 @@ public void testRetryableException() throws Exception { .andReturn(responsePb); EasyMock.replay(rpcFactoryMock, rpcMock); DatastoreOptions options = this.options.toBuilder() - .retryParams(RetryParams.getDefaultInstance()) + .retryParams(RetryParams.defaultInstance()) .serviceRpcFactory(rpcFactoryMock) .build(); Datastore datastore = options.service(); @@ -784,7 +784,7 @@ public void testRuntimeException() throws Exception { .andThrow(new RuntimeException(exceptionMessage)); EasyMock.replay(rpcFactoryMock, rpcMock); DatastoreOptions options = this.options.toBuilder() - .retryParams(RetryParams.getDefaultInstance()) + .retryParams(RetryParams.defaultInstance()) .serviceRpcFactory(rpcFactoryMock) .build(); Datastore datastore = options.service(); diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 1ad690938ef5..32e14fb47ea0 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -143,7 +143,7 @@ public void testServiceOptions() throws Exception { options = options.toBuilder() .namespace("ns1") - .retryParams(RetryParams.getDefaultInstance()) + .retryParams(RetryParams.defaultInstance()) .authCredentials(AuthCredentials.noCredentials()) .force(true) .build(); diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java index b7a36de7589e..0e7d6bbbf3ea 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -18,7 +18,6 @@ import com.google.gcloud.AuthCredentials; import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; -import com.google.gcloud.Page; import com.google.gcloud.RetryParams; import com.google.gcloud.spi.StorageRpc.Tuple; import com.google.gcloud.storage.Blob; @@ -546,7 +545,7 @@ public static void main(String... args) throws Exception { return; } StorageOptions.Builder optionsBuilder = - StorageOptions.builder().retryParams(RetryParams.getDefaultInstance()); + StorageOptions.builder().retryParams(RetryParams.defaultInstance()); StorageAction action; String actionName; if (args.length >= 2 && !ACTIONS.containsKey(args[0])) { diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java index 1da45fae466e..3c3d1aebb3df 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelperTest.java @@ -159,11 +159,11 @@ public void testCreateFromStream() { assertEquals(PROJECT_ID, options.projectId()); assertEquals(60000, options.connectTimeout()); assertEquals(60000, options.readTimeout()); - assertEquals(10, options.retryParams().getRetryMaxAttempts()); - assertEquals(6, options.retryParams().getRetryMinAttempts()); - assertEquals(30000, options.retryParams().getMaxRetryDelayMillis()); - assertEquals(120000, options.retryParams().getTotalRetryPeriodMillis()); - assertEquals(250, options.retryParams().getInitialRetryDelayMillis()); + assertEquals(10, options.retryParams().retryMaxAttempts()); + assertEquals(6, options.retryParams().retryMinAttempts()); + assertEquals(30000, options.retryParams().maxRetryDelayMillis()); + assertEquals(120000, options.retryParams().totalRetryPeriodMillis()); + assertEquals(250, options.retryParams().initialRetryDelayMillis()); } @Test diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java index a125a64df6d6..2d80191aeb2d 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java @@ -82,7 +82,7 @@ public void testServiceOptions() throws Exception { options = options.toBuilder() .projectId("p2") - .retryParams(RetryParams.getDefaultInstance()) + .retryParams(RetryParams.defaultInstance()) .authCredentials(AuthCredentials.noCredentials()) .pathDelimiter(":") .build(); @@ -110,7 +110,7 @@ public void testModelAndRequests() throws Exception { public void testReadChannelState() throws IOException, ClassNotFoundException { StorageOptions options = StorageOptions.builder() .projectId("p2") - .retryParams(RetryParams.getDefaultInstance()) + .retryParams(RetryParams.defaultInstance()) .authCredentials(AuthCredentials.noCredentials()) .build(); BlobReadChannel reader = @@ -126,7 +126,7 @@ public void testReadChannelState() throws IOException, ClassNotFoundException { public void testWriteChannelState() throws IOException, ClassNotFoundException { StorageOptions options = StorageOptions.builder() .projectId("p2") - .retryParams(RetryParams.getDefaultInstance()) + .retryParams(RetryParams.defaultInstance()) .authCredentials(AuthCredentials.noCredentials()) .build(); BlobWriteChannelImpl writer = new BlobWriteChannelImpl( diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java index d5e2f8de7397..f07c7000813e 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java @@ -1202,7 +1202,7 @@ public void testRetryableException() { .andThrow(new StorageException(500, "InternalError", true)) .andReturn(BLOB_INFO1.toPb()); EasyMock.replay(storageRpcMock); - storage = options.toBuilder().retryParams(RetryParams.getDefaultInstance()).build().service(); + storage = options.toBuilder().retryParams(RetryParams.defaultInstance()).build().service(); BlobInfo readBlob = storage.get(blob); assertEquals(BLOB_INFO1, readBlob); } @@ -1214,7 +1214,7 @@ public void testNonRetryableException() { EasyMock.expect(storageRpcMock.get(blob.toPb(), EMPTY_RPC_OPTIONS)) .andThrow(new StorageException(501, exceptionMessage, false)); EasyMock.replay(storageRpcMock); - storage = options.toBuilder().retryParams(RetryParams.getDefaultInstance()).build().service(); + storage = options.toBuilder().retryParams(RetryParams.defaultInstance()).build().service(); thrown.expect(StorageException.class); thrown.expectMessage(exceptionMessage); storage.get(blob); @@ -1227,7 +1227,7 @@ public void testRuntimeException() { EasyMock.expect(storageRpcMock.get(blob.toPb(), EMPTY_RPC_OPTIONS)) .andThrow(new RuntimeException(exceptionMessage)); EasyMock.replay(storageRpcMock); - storage = options.toBuilder().retryParams(RetryParams.getDefaultInstance()).build().service(); + storage = options.toBuilder().retryParams(RetryParams.defaultInstance()).build().service(); thrown.expect(StorageException.class); thrown.expectMessage(exceptionMessage); storage.get(blob); From b28b3079539e2c1635da8a2e795055c05f2f5a07 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 21:55:39 +0100 Subject: [PATCH 30/94] Make ExceptionHandler conform with our code style --- .../src/main/java/com/google/gcloud/ExceptionHandler.java | 6 +++--- .../src/main/java/com/google/gcloud/RetryHelper.java | 2 +- .../test/java/com/google/gcloud/ExceptionHandlerTest.java | 2 +- .../src/test/java/com/google/gcloud/RetryHelperTest.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java b/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java index a0fab3dca566..c1f068594443 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java @@ -231,11 +231,11 @@ void verifyCaller(Callable callable) { } } - public Set> getRetriableExceptions() { + public Set> retriableExceptions() { return retriableExceptions; } - public Set> getNonRetriableExceptions() { + public Set> nonRetriableExceptions() { return nonRetriableExceptions; } @@ -262,7 +262,7 @@ boolean shouldRetry(Exception ex) { /** * Returns an instance which retry any checked exception and abort on any runtime exception. */ - public static ExceptionHandler getDefaultInstance() { + public static ExceptionHandler defaultInstance() { return DEFAULT_INSTANCE; } diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/RetryHelper.java b/gcloud-java-core/src/main/java/com/google/gcloud/RetryHelper.java index 90f403758e45..9b9c1f6a3124 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/RetryHelper.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/RetryHelper.java @@ -229,7 +229,7 @@ private static long getExponentialValue(long initialDelay, double backoffFactor, public static V runWithRetries(Callable callable) throws RetryHelperException { return runWithRetries(callable, RetryParams.defaultInstance(), - ExceptionHandler.getDefaultInstance()); + ExceptionHandler.defaultInstance()); } public static V runWithRetries(Callable callable, RetryParams params, diff --git a/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java index c182515dbb16..cedc995ddbd0 100644 --- a/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java +++ b/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java @@ -82,7 +82,7 @@ public Object call() throws Error { } // using default exception handler (retry upon any non-runtime exceptions) - ExceptionHandler handler = ExceptionHandler.getDefaultInstance(); + ExceptionHandler handler = ExceptionHandler.defaultInstance(); assertValidCallable(new A(), handler); assertValidCallable(new B(), handler); assertValidCallable(new C(), handler); diff --git a/gcloud-java-core/src/test/java/com/google/gcloud/RetryHelperTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/RetryHelperTest.java index bf11e744dbbf..9a7cc2104f4a 100644 --- a/gcloud-java-core/src/test/java/com/google/gcloud/RetryHelperTest.java +++ b/gcloud-java-core/src/test/java/com/google/gcloud/RetryHelperTest.java @@ -124,7 +124,7 @@ public void testTriesAtLeastMinTimes() { } return timesCalled; } - }, params, ExceptionHandler.getDefaultInstance()); + }, params, ExceptionHandler.defaultInstance()); assertEquals(timesToFail + 1, attempted); assertNull(RetryHelper.getContext()); } From abc11cea3aedd69e47ae88fa50fd5ad96d16823e Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 11 Nov 2015 23:06:01 +0100 Subject: [PATCH 31/94] Use DatastoreOptions.defaultInstance in site example --- src/site/resources/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/site/resources/index.html b/src/site/resources/index.html index 25c769db2fbf..3ed496b5bbb3 100644 --- a/src/site/resources/index.html +++ b/src/site/resources/index.html @@ -143,7 +143,7 @@

Example: Retrieve Datastore Entries

// Authentication is automatic inside Google Compute Engine // and Google App Engine. -DatastoreOptions options = DatastoreOptions.builder().build(); +DatastoreOptions options = DatastoreOptions.defaultInstance(); Datastore datastore = options.service(); KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND); Key key = keyFactory.newKey(keyName); From 7193d0d02a252f9c8cb0ece901fc6e16e59f5413 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 12 Nov 2015 07:58:35 +0100 Subject: [PATCH 32/94] More detailed javadoc for BlobInfo --- .../com/google/gcloud/storage/BlobInfo.java | 67 +++++++++++++++---- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java index 9662635160e8..65b87498b6cc 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java @@ -138,6 +138,8 @@ Builder id(String id) { /** * Sets the blob's data content type. + * + * @see Content-Type */ public Builder contentType(String contentType) { this.contentType = firstNonNull(contentType, Data.nullOf(String.class)); @@ -146,6 +148,8 @@ public Builder contentType(String contentType) { /** * Sets the blob's data content disposition. + * + * @see Content-Disposition */ public Builder contentDisposition(String contentDisposition) { this.contentDisposition = firstNonNull(contentDisposition, Data.nullOf(String.class)); @@ -154,6 +158,8 @@ public Builder contentDisposition(String contentDisposition) { /** * Sets the blob's data content language. + * + * @see Content-Language */ public Builder contentLanguage(String contentLanguage) { this.contentLanguage = firstNonNull(contentLanguage, Data.nullOf(String.class)); @@ -162,6 +168,8 @@ public Builder contentLanguage(String contentLanguage) { /** * Sets the blob's data content encoding. + * + * @see Content-Encoding */ public Builder contentEncoding(String contentEncoding) { this.contentEncoding = firstNonNull(contentEncoding, Data.nullOf(String.class)); @@ -175,6 +183,8 @@ Builder componentCount(Integer componentCount) { /** * Sets the blob's data cache control. + * + * @see Cache-Control */ public Builder cacheControl(String cacheControl) { this.cacheControl = firstNonNull(cacheControl, Data.nullOf(String.class)); @@ -214,6 +224,9 @@ Builder selfLink(String selfLink) { /** * Sets the MD5 hash of blob's data. MD5 value must be encoded in base64. + * + * @see + * Hashes and ETags: Best Practices */ public Builder md5(String md5) { this.md5 = firstNonNull(md5, Data.nullOf(String.class)); @@ -221,8 +234,12 @@ public Builder md5(String md5) { } /** - * Sets the CRC32C checksum of blob's data. CRC32C value must be encoded in base64 in big-endian - * order. + * Sets the CRC32C checksum of blob's data as described in + * RFC 4960, Appendix B; encoded in + * base64 in big-endian order. + * + * @see + * Hashes and ETags: Best Practices */ public Builder crc32c(String crc32c) { this.crc32c = firstNonNull(crc32c, Data.nullOf(String.class)); @@ -235,7 +252,7 @@ Builder mediaLink(String mediaLink) { } /** - * Sets the blob's metadata. + * Sets the blob's user provided metadata. */ public Builder metadata(Map metadata) { this.metadata = metadata != null @@ -326,6 +343,8 @@ public String name() { /** * Returns the blob's data cache control. + * + * @see Cache-Control */ public String cacheControl() { return Data.isNull(cacheControl) ? null : cacheControl; @@ -342,14 +361,16 @@ public List acl() { } /** - * Returns the blob's owner. + * Returns the blob's owner. This will always be the uploader of the blob. */ public Acl.Entity owner() { return owner; } /** - * Returns the blob's data size in bytes. + * Returns the content length of the data in bytes. + * + * @see Content-Length */ public Long size() { return size; @@ -357,6 +378,8 @@ public Long size() { /** * Returns the blob's data content type. + * + * @see Content-Type */ public String contentType() { return Data.isNull(contentType) ? null : contentType; @@ -364,6 +387,8 @@ public String contentType() { /** * Returns the blob's data content encoding. + * + * @see Content-Encoding */ public String contentEncoding() { return Data.isNull(contentEncoding) ? null : contentEncoding; @@ -371,6 +396,8 @@ public String contentEncoding() { /** * Returns the blob's data content disposition. + * + * @see Content-Disposition */ public String contentDisposition() { return Data.isNull(contentDisposition) ? null : contentDisposition; @@ -378,14 +405,18 @@ public String contentDisposition() { /** * Returns the blob's data content language. + * + * @see Content-Language */ public String contentLanguage() { return Data.isNull(contentLanguage) ? null : contentLanguage; } /** - * Returns the number of components that make up this object. Components are accumulated through - * the {@link Storage#compose(Storage.ComposeRequest)} operation. + * Returns the number of components that make up this blob. Components are accumulated through + * the {@link Storage#compose(Storage.ComposeRequest)} operation and are limited to a count of + * 1024, counting 1 for each non-composite component blob and componentCount for each composite + * component blob. This value is set only for composite blobs. * * @see Component Count * Property @@ -395,7 +426,9 @@ public Integer componentCount() { } /** - * Returns blob resource's entity tag. + * Returns HTTP 1.1 Entity tag for the blob. + * + * @see Entity Tags */ public String etag() { return etag; @@ -410,13 +443,21 @@ public String selfLink() { /** * Returns the MD5 hash of blob's data encoded in base64. + * + * @see + * Hashes and ETags: Best Practices */ public String md5() { return Data.isNull(md5) ? null : md5; } /** - * Returns the CRC32C checksum of blob's data encoded in base64 in big-endian order. + * Returns the CRC32C checksum of blob's data as described in + * RFC 4960, Appendix B; encoded in + * base64 in big-endian order. + * + * @see + * Hashes and ETags: Best Practices */ public String crc32c() { return Data.isNull(crc32c) ? null : crc32c; @@ -430,21 +471,23 @@ public String mediaLink() { } /** - * Returns blob's metadata. + * Returns blob's user provided metadata. */ public Map metadata() { return metadata == null || Data.isNull(metadata) ? null : Collections.unmodifiableMap(metadata); } /** - * Returns blob's data generation. + * Returns blob's data generation. Used for blob versioning. */ public Long generation() { return generation; } /** - * Returns blob's metageneration. + * Returns blob's metageneration. Used for preconditions and for detecting changes in metadata. + * A metageneration number is only meaningful in the context of a particular generation of a + * particular blob. */ public Long metageneration() { return metageneration; From 5f9eaa4cab417c161b436bb5c5da4046b056b3e6 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 12 Nov 2015 07:59:38 +0100 Subject: [PATCH 33/94] Better javadoc for (Bucket/Blob)ListOptions.prefix --- .../src/main/java/com/google/gcloud/storage/Storage.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 128099b7eab3..2e95e69aa445 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -594,7 +594,8 @@ public static BucketListOption startPageToken(String pageToken) { } /** - * Returns an option to set a prefix for listed buckets names. + * Returns an option to set a prefix to filter results to buckets whose names begin with this + * prefix. */ public static BucketListOption prefix(String prefix) { return new BucketListOption(StorageRpc.Option.PREFIX, prefix); @@ -639,7 +640,8 @@ public static BlobListOption startPageToken(String pageToken) { } /** - * Returns an option to set a prefix for listed blobs names. + * Returns an option to set a prefix to filter results to blobs whose names begin with this + * prefix. */ public static BlobListOption prefix(String prefix) { return new BlobListOption(StorageRpc.Option.PREFIX, prefix); From b8defb8e585468e0b95142080248a9f00be87c18 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 12 Nov 2015 07:30:55 +0100 Subject: [PATCH 34/94] Better javadoc for BucketInfo --- .../com/google/gcloud/storage/BucketInfo.java | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java index 3f21f74dcd66..b40e023cc47e 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -300,7 +300,7 @@ public static class IsLiveDeleteRule extends DeleteRule { * Creates an {@code IsLiveDeleteRule} object. * * @param isLive if set to {@code true} live blobs meet the delete condition. If set to - * {@code false} delete condition is met by archived objects. + * {@code false} delete condition is met by archived blobs. */ public IsLiveDeleteRule(boolean isLive) { super(Type.IS_LIVE); @@ -362,7 +362,8 @@ Builder selfLink(String selfLink) { } /** - * Sets whether blob versioning should be enabled or not for the bucket. + * Sets whether versioning should be enabled for this bucket. When set to true, versioning is + * fully enabled. */ public Builder versioningEnabled(Boolean enable) { this.versioningEnabled = firstNonNull(enable, Data.nullOf(Boolean.class)); @@ -370,7 +371,8 @@ public Builder versioningEnabled(Boolean enable) { } /** - * Sets the bucket's website index page. + * Sets the bucket's website index page. Behaves as the bucket's directory index where missing + * blobs are treated as potential directories. */ public Builder indexPage(String indexPage) { this.indexPage = indexPage; @@ -378,7 +380,7 @@ public Builder indexPage(String indexPage) { } /** - * Sets the bucket's website page to return when a resource is not found. + * Sets the custom object to return when a requested resource is not found. */ public Builder notFoundPage(String notFoundPage) { this.notFoundPage = notFoundPage; @@ -386,7 +388,7 @@ public Builder notFoundPage(String notFoundPage) { } /** - * Sets bucket's delete rules. + * Sets the bucket's lifecycle configuration as a number of delete rules. * * @see Lifecycle Management */ @@ -396,10 +398,9 @@ public Builder deleteRules(Iterable rules) { } /** - * Sets the bucket's storage class. A list of supported values is available - * here. - * - * @see Storage Classes + * Sets the bucket's storage class. This defines how blobs in the bucket are stored and + * determines the SLA and the cost of storage. A list of supported values is available + * here. */ public Builder storageClass(String storageClass) { this.storageClass = storageClass; @@ -407,7 +408,8 @@ public Builder storageClass(String storageClass) { } /** - * Sets the bucket's location. A list of supported values is available + * Sets the bucket's location. Data for blobs in the bucket resides in physical storage within + * this region. A list of supported values is available * here. */ public Builder location(String location) { @@ -507,7 +509,7 @@ public String name() { } /** - * Returns the bucket's owner. + * Returns the bucket's owner. This is always the project team's owner group. */ public Entity owner() { return owner; @@ -521,35 +523,40 @@ public String selfLink() { } /** - * Returns {@code true} if blob versioning is enabled for this bucket, {@code false} otherwise. + * Returns {@code true} if versioning is fully enabled for this bucket, {@code false} otherwise. */ public Boolean versioningEnabled() { return Data.isNull(versioningEnabled) ? null : versioningEnabled; } /** - * Returns bucket's website index page. + * Returns bucket's website index page. Behaves as the bucket's directory index where missing + * blobs are treated as potential directories. */ public String indexPage() { return indexPage; } /** - * Returns bucket's website not found page, used we a resource could not be found. + * Returns the custom object to return when a requested resource is not found. */ public String notFoundPage() { return notFoundPage; } /** - * Returns bucket's delete rules. + * Returns bucket's lifecycle configuration as a number of delete rules. + * + * @see Lifecycle Management */ public List deleteRules() { return deleteRules; } /** - * Returns bucket resource's entity tag. + * Returns HTTP 1.1 Entity tag for the bucket. + * + * @see Entity Tags */ public String etag() { return etag; @@ -570,7 +577,8 @@ public Long metageneration() { } /** - * Returns the bucket's location. + * Returns the bucket's location. Data for blobs in the bucket resides in physical storage within + * this region. * * @see Bucket Locations */ @@ -579,7 +587,8 @@ public String location() { } /** - * Returns the bucket's storage class. + * Returns the bucket's storage class. This defines how blobs in the bucket are stored and + * determines the SLA and the cost of storage. * * @see Storage Classes */ @@ -589,6 +598,9 @@ public String storageClass() { /** * Returns the bucket's Cross-Origin Resource Sharing (CORS) configuration. + * + * @see + * Cross-Origin Resource Sharing (CORS) */ public List cors() { return cors; From da0ea589ff5362b261e92d1b755a16699abacd42 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 12 Nov 2015 07:34:54 +0100 Subject: [PATCH 35/94] Removed unused import from BucketInfo --- .../src/main/java/com/google/gcloud/storage/BucketInfo.java | 1 - 1 file changed, 1 deletion(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java index b40e023cc47e..d5a382446709 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java @@ -33,7 +33,6 @@ import com.google.common.base.Function; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.gcloud.storage.Acl.Entity; import java.io.IOException; From 2ffac2db4744375d2c5cbd8e1f84475ffcca4347 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 12 Nov 2015 12:34:45 +0100 Subject: [PATCH 36/94] Move generation from BlobInfo to BlobId - Add generation to BlobId and remove from BlobInfo, update tests - Add generationMatch() and generationNotMatch() methods to (BlobSource/BlobGet)Option - Add setGeneration method to set generation value in empty generation options from BlobId - Add support for empty generation options in storage.get - Add support for empty generation options in storage.readAllBytes - Add support for empty generation options in storage.reader - Add support for empty generation options in storage.delete - Add support for empty generation options in BatchRequest - Add support for empty generation options in CopyRequest - Update/and unit and integration tests --- .../google/gcloud/storage/BatchRequest.java | 4 +- .../com/google/gcloud/storage/BlobId.java | 41 +++++- .../com/google/gcloud/storage/BlobInfo.java | 29 ++--- .../com/google/gcloud/storage/Storage.java | 83 +++++++++++- .../google/gcloud/storage/StorageImpl.java | 17 ++- .../gcloud/storage/BatchRequestTest.java | 17 ++- .../google/gcloud/storage/BlobInfoTest.java | 7 +- .../google/gcloud/storage/ITStorageTest.java | 121 ++++++++++++++---- .../gcloud/storage/StorageImplTest.java | 78 ++++++++++- 9 files changed, 322 insertions(+), 75 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java index bf77c731754e..ae48c5790a77 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java @@ -58,7 +58,7 @@ public Builder delete(String bucket, String blob, BlobSourceOption... options) { * Delete the given blob. */ public Builder delete(BlobId blob, BlobSourceOption... options) { - toDelete.put(blob, Lists.newArrayList(options)); + toDelete.put(blob, Lists.newArrayList(BlobSourceOption.setGeneration(blob, options))); return this; } @@ -82,7 +82,7 @@ public Builder get(String bucket, String blob, BlobGetOption... options) { * Retrieve metadata for the given blob. */ public Builder get(BlobId blob, BlobGetOption... options) { - toGet.put(blob, Lists.newArrayList(options)); + toGet.put(blob, Lists.newArrayList(BlobGetOption.setGeneration(blob, options))); return this; } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobId.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobId.java index d1209826cc3e..d30003d632db 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobId.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobId.java @@ -25,17 +25,21 @@ import java.util.Objects; /** - * Google Storage object identifier. + * Google Storage Object identifier. A {@code BlobId} object includes the name of the containing + * bucket, the blob's name and possibly the blob's generation. If {@link #generation()} is + * {@code null} the identifier refers to the latest blob's generation. */ public final class BlobId implements Serializable { private static final long serialVersionUID = -6156002883225601925L; private final String bucket; private final String name; + private final Long generation; - private BlobId(String bucket, String name) { + private BlobId(String bucket, String name, Long generation) { this.bucket = bucket; this.name = name; + this.generation = generation; } /** @@ -52,43 +56,66 @@ public String name() { return name; } + /** + * Returns blob's data generation. Used for versioning. + */ + public Long generation() { + return generation; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) .add("bucket", bucket()) .add("name", name()) + .add("generation", generation()) .toString(); } @Override public int hashCode() { - return Objects.hash(bucket, name); + return Objects.hash(bucket, name, generation); } @Override public boolean equals(Object obj) { return obj instanceof BlobId && Objects.equals(bucket, ((BlobId) obj).bucket) - && Objects.equals(name, ((BlobId) obj).name); + && Objects.equals(name, ((BlobId) obj).name) + && Objects.equals(generation, ((BlobId) obj).generation); } StorageObject toPb() { StorageObject storageObject = new StorageObject(); storageObject.setBucket(bucket); storageObject.setName(name); + storageObject.setGeneration(generation); return storageObject; } /** - * Creates a blob identifier. + * Creates a blob identifier. Generation is set to {@code null}. * * @param bucket the name of the bucket that contains the blob * @param name the name of the blob */ public static BlobId of(String bucket, String name) { - return new BlobId(checkNotNull(bucket), checkNotNull(name)); + return new BlobId(checkNotNull(bucket), checkNotNull(name), null); + } + + /** + * Creates a {@code BlobId} object. + * + * @param bucket name of the containing bucket + * @param name blob's name + * @param generation blob's data generation, used for versioning. If {@code null} the identifier + * refers to the latest blob's generation + */ + public static BlobId of(String bucket, String name, Long generation) { + return new BlobId(checkNotNull(bucket), checkNotNull(name), generation); } static BlobId fromPb(StorageObject storageObject) { - return BlobId.of(storageObject.getBucket(), storageObject.getName()); + return BlobId.of(storageObject.getBucket(), storageObject.getName(), + storageObject.getGeneration()); } } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java index 65b87498b6cc..29cf16dcd617 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java @@ -76,7 +76,6 @@ public StorageObject apply(BlobInfo blobInfo) { private final String crc32c; private final String mediaLink; private final Map metadata; - private final Long generation; private final Long metageneration; private final Long deleteTime; private final Long updateTime; @@ -116,7 +115,6 @@ public static final class Builder { private String crc32c; private String mediaLink; private Map metadata; - private Long generation; private Long metageneration; private Long deleteTime; private Long updateTime; @@ -260,11 +258,6 @@ public Builder metadata(Map metadata) { return this; } - Builder generation(Long generation) { - this.generation = generation; - return this; - } - Builder metageneration(Long metageneration) { this.metageneration = metageneration; return this; @@ -307,7 +300,6 @@ private BlobInfo(Builder builder) { crc32c = builder.crc32c; mediaLink = builder.mediaLink; metadata = builder.metadata; - generation = builder.generation; metageneration = builder.metageneration; deleteTime = builder.deleteTime; updateTime = builder.updateTime; @@ -481,7 +473,7 @@ public Map metadata() { * Returns blob's data generation. Used for blob versioning. */ public Long generation() { - return generation; + return blobId().generation(); } /** @@ -514,7 +506,6 @@ public Builder toBuilder() { return new Builder() .blobId(blobId) .id(id) - .generation(generation) .cacheControl(cacheControl) .contentEncoding(contentEncoding) .contentType(contentType) @@ -540,6 +531,7 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("bucket", bucket()) .add("name", name()) + .add("generation", generation()) .add("size", size()) .add("content-type", contentType()) .add("metadata", metadata()) @@ -590,7 +582,6 @@ public ObjectAccessControl apply(Acl acl) { storageObject.setContentEncoding(contentEncoding); storageObject.setCrc32c(crc32c); storageObject.setContentType(contentType); - storageObject.setGeneration(generation); storageObject.setMd5Hash(md5); storageObject.setMediaLink(mediaLink); storageObject.setMetageneration(metageneration); @@ -618,8 +609,19 @@ public static Builder builder(String bucket, String name) { } /** - * Returns a {@code BlobInfo} builder where blob identity is set to the provided value. + * Returns a {@code BlobInfo} builder where blob identity is set using the provided values. + */ + public static Builder builder(BucketInfo bucketInfo, String name, Long generation) { + return builder(bucketInfo.name(), name, generation); + } + + /** + * Returns a {@code BlobInfo} builder where blob identity is set using the provided values. */ + public static Builder builder(String bucket, String name, Long generation) { + return new Builder().blobId(BlobId.of(bucket, name, generation)); + } + public static Builder builder(BlobId blobId) { return new Builder().blobId(blobId); } @@ -638,9 +640,6 @@ static BlobInfo fromPb(StorageObject storageObject) { if (storageObject.getContentType() != null) { builder.contentType(storageObject.getContentType()); } - if (storageObject.getGeneration() != null) { - builder.generation(storageObject.getGeneration()); - } if (storageObject.getMd5Hash() != null) { builder.md5(storageObject.getMd5Hash()); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 2e95e69aa445..4210a278c413 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -473,14 +473,32 @@ class BlobSourceOption extends Option { private static final long serialVersionUID = -3712768261070182991L; - private BlobSourceOption(StorageRpc.Option rpcOption, long value) { + private BlobSourceOption(StorageRpc.Option rpcOption, Long value) { super(rpcOption, value); } /** * Returns an option for blob's data generation match. If this option is used the request will - * fail if blob's generation does not match the provided value. + * fail if blob's generation does not match. The generation value to compare with the actual + * blob's generation is taken from a source {@link BlobId} object. When this option is passed + * to a {@link Storage} method and {@link BlobId#generation()} is {@code null} or no + * {@link BlobId} is provided an exception is thrown. */ + public static BlobSourceOption generationMatch() { + return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_MATCH, null); + } + + /** + * Returns an option for blob's data generation mismatch. If this option is used the request + * will fail if blob's generation matches. The generation value to compare with the actual + * blob's generation is taken from a source {@link BlobId} object. When this option is passed + * to a {@link Storage} method and {@link BlobId#generation()} is {@code null} or no + * {@link BlobId} is provided an exception is thrown. + */ + public static BlobSourceOption generationNotMatch() { + return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH, null); + } + public static BlobSourceOption generationMatch(long generation) { return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_MATCH, generation); } @@ -508,6 +526,26 @@ public static BlobSourceOption metagenerationMatch(long metageneration) { public static BlobSourceOption metagenerationNotMatch(long metageneration) { return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration); } + + static BlobSourceOption[] setGeneration(BlobId blobId, Iterable options) { + return setGeneration(blobId, Iterables.toArray(options, BlobSourceOption.class)); + } + + static BlobSourceOption[] setGeneration(BlobId blobId, BlobSourceOption... options) { + BlobSourceOption[] updatedOptions = new BlobSourceOption[options.length]; + int index = 0; + for (BlobSourceOption option : options) { + if ((option.rpcOption() == StorageRpc.Option.IF_GENERATION_MATCH + || option.rpcOption() == StorageRpc.Option.IF_GENERATION_NOT_MATCH) + && option.value() == null) { + updatedOptions[index] = new BlobSourceOption(option.rpcOption(), blobId.generation()); + } else { + updatedOptions[index] = option; + } + index++; + } + return updatedOptions; + } } /** @@ -517,7 +555,7 @@ class BlobGetOption extends Option { private static final long serialVersionUID = 803817709703661480L; - private BlobGetOption(StorageRpc.Option rpcOption, long value) { + private BlobGetOption(StorageRpc.Option rpcOption, Long value) { super(rpcOption, value); } @@ -527,8 +565,26 @@ private BlobGetOption(StorageRpc.Option rpcOption, String value) { /** * Returns an option for blob's data generation match. If this option is used the request will - * fail if blob's generation does not match the provided value. + * fail if blob's generation does not match. The generation value to compare with the actual + * blob's generation is taken from a source {@link BlobId} object. When this option is passed + * to a {@link Storage} method and {@link BlobId#generation()} is {@code null} or no + * {@link BlobId} is provided an exception is thrown. + */ + public static BlobGetOption generationMatch() { + return new BlobGetOption(StorageRpc.Option.IF_GENERATION_MATCH, (Long) null); + } + + /** + * Returns an option for blob's data generation mismatch. If this option is used the request + * will fail if blob's generation matches. The generation value to compare with the actual + * blob's generation is taken from a source {@link BlobId} object. When this option is passed + * to a {@link Storage} method and {@link BlobId#generation()} is {@code null} or no + * {@link BlobId} is provided an exception is thrown. */ + public static BlobGetOption generationNotMatch() { + return new BlobGetOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH, (Long) null); + } + public static BlobGetOption generationMatch(long generation) { return new BlobGetOption(StorageRpc.Option.IF_GENERATION_MATCH, generation); } @@ -566,6 +622,22 @@ public static BlobGetOption metagenerationNotMatch(long metageneration) { public static BlobGetOption fields(BlobField... fields) { return new BlobGetOption(StorageRpc.Option.FIELDS, BlobField.selector(fields)); } + + static BlobGetOption[] setGeneration(BlobId blobId, BlobGetOption... options) { + BlobGetOption[] updatedOptions = new BlobGetOption[options.length]; + int index = 0; + for (BlobGetOption option : options) { + if ((option.rpcOption() == StorageRpc.Option.IF_GENERATION_MATCH + || option.rpcOption() == StorageRpc.Option.IF_GENERATION_NOT_MATCH) + && option.value() == null) { + updatedOptions[index] = new BlobGetOption(option.rpcOption(), blobId.generation()); + } else { + updatedOptions[index] = option; + } + index++; + } + return updatedOptions; + } } /** @@ -1019,7 +1091,8 @@ public CopyRequest build() { private CopyRequest(Builder builder) { source = checkNotNull(builder.source); - sourceOptions = ImmutableList.copyOf(builder.sourceOptions); + sourceOptions = ImmutableList.copyOf( + BlobSourceOption.setGeneration(source, builder.sourceOptions)); target = checkNotNull(builder.target); targetOptions = ImmutableList.copyOf(builder.targetOptions); megabytesCopiedPerChunk = builder.megabytesCopiedPerChunk; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index 4c85113e940e..b4da9e14fd97 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -199,7 +199,8 @@ public BlobInfo get(String bucket, String blob, BlobGetOption... options) { @Override public BlobInfo get(BlobId blob, BlobGetOption... options) { final StorageObject storedObject = blob.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = + optionMap(BlobGetOption.setGeneration(blob, options)); try { StorageObject storageObject = runWithRetries(new Callable() { @Override @@ -405,7 +406,8 @@ public boolean delete(String bucket, String blob, BlobSourceOption... options) { @Override public boolean delete(BlobId blob, BlobSourceOption... options) { final StorageObject storageObject = blob.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = + optionMap(BlobSourceOption.setGeneration(blob, options)); try { return runWithRetries(new Callable() { @Override @@ -428,8 +430,9 @@ public BlobInfo compose(final ComposeRequest composeRequest) { final List sources = Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size()); for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) { - sources.add(BlobInfo.builder(composeRequest.target().bucket(), sourceBlob.name()) - .generation(sourceBlob.generation()).build().toPb()); + sources.add(BlobInfo.builder( + BlobId.of(composeRequest.target().bucket(), sourceBlob.name(), sourceBlob.generation())) + .build().toPb()); } final StorageObject target = composeRequest.target().toPb(); final Map targetOptions = optionMap(composeRequest.target().generation(), @@ -476,7 +479,8 @@ public byte[] readAllBytes(String bucket, String blob, BlobSourceOption... optio @Override public byte[] readAllBytes(BlobId blob, BlobSourceOption... options) { final StorageObject storageObject = blob.toPb(); - final Map optionsMap = optionMap(options); + final Map optionsMap = + optionMap(BlobSourceOption.setGeneration(blob, options)); try { return runWithRetries(new Callable() { @Override @@ -557,7 +561,8 @@ public BlobReadChannel reader(String bucket, String blob, BlobSourceOption... op @Override public BlobReadChannel reader(BlobId blob, BlobSourceOption... options) { - Map optionsMap = optionMap(options); + Map optionsMap = + optionMap(BlobSourceOption.setGeneration(blob, options)); return new BlobReadChannelImpl(options(), blob, optionsMap); } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java index 600c8af0d554..020bdf5ed404 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java @@ -37,12 +37,12 @@ public class BatchRequestTest { @Test public void testBatchRequest() { BatchRequest request = BatchRequest.builder() - .delete("b1", "o1") + .delete(BlobId.of("b1", "o1", 1L), BlobSourceOption.generationMatch()) .delete("b1", "o2", BlobSourceOption.generationMatch(1), BlobSourceOption.metagenerationMatch(2)) .update(BlobInfo.builder("b2", "o1").build(), BlobTargetOption.predefinedAcl(PUBLIC_READ)) .update(BlobInfo.builder("b2", "o2").build()) - .get("b3", "o1") + .get(BlobId.of("b3", "o1", 1L), BlobGetOption.generationMatch()) .get("b3", "o2", BlobGetOption.generationMatch(1)) .get("b3", "o3") .build(); @@ -50,11 +50,15 @@ public void testBatchRequest() { Iterator>> deletes = request .toDelete().entrySet().iterator(); Entry> delete = deletes.next(); - assertEquals(BlobId.of("b1", "o1"), delete.getKey()); - assertTrue(Iterables.isEmpty(delete.getValue())); + assertEquals(BlobId.of("b1", "o1", 1L), delete.getKey()); + assertEquals(1, Iterables.size(delete.getValue())); + assertEquals(BlobSourceOption.generationMatch(1L), Iterables.getFirst(delete.getValue(), null)); delete = deletes.next(); assertEquals(BlobId.of("b1", "o2"), delete.getKey()); assertEquals(2, Iterables.size(delete.getValue())); + assertEquals(BlobSourceOption.generationMatch(1L), Iterables.getFirst(delete.getValue(), null)); + assertEquals(BlobSourceOption.metagenerationMatch(2L), + Iterables.get(delete.getValue(), 1, null)); assertFalse(deletes.hasNext()); Iterator>> updates = request @@ -71,8 +75,9 @@ public void testBatchRequest() { Iterator>> gets = request.toGet().entrySet().iterator(); Entry> get = gets.next(); - assertEquals(BlobId.of("b3", "o1"), get.getKey()); - assertTrue(Iterables.isEmpty(get.getValue())); + assertEquals(BlobId.of("b3", "o1", 1L), get.getKey()); + assertEquals(1, Iterables.size(get.getValue())); + assertEquals(BlobGetOption.generationMatch(1), Iterables.getFirst(get.getValue(), null)); get = gets.next(); assertEquals(BlobId.of("b3", "o2"), get.getKey()); assertEquals(1, Iterables.size(get.getValue())); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java index 70560b0c9a9e..7214170afe9a 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java @@ -55,7 +55,7 @@ public class BlobInfoTest { private static final String SELF_LINK = "http://storage/b/n"; private static final Long SIZE = 1024L; private static final Long UPDATE_TIME = DELETE_TIME - 1L; - private static final BlobInfo BLOB_INFO = BlobInfo.builder("b", "n") + private static final BlobInfo BLOB_INFO = BlobInfo.builder("b", "n", GENERATION) .acl(ACL) .componentCount(COMPONENT_COUNT) .contentType(CONTENT_TYPE) @@ -66,7 +66,6 @@ public class BlobInfoTest { .crc32c(CRC32) .deleteTime(DELETE_TIME) .etag(ETAG) - .generation(GENERATION) .id(ID) .md5(MD5) .mediaLink(MEDIA_LINK) @@ -85,7 +84,7 @@ public void testToBuilder() { assertEquals("n2", blobInfo.name()); assertEquals("b2", blobInfo.bucket()); assertEquals(Long.valueOf(200), blobInfo.size()); - blobInfo = blobInfo.toBuilder().blobId(BlobId.of("b", "n")).size(SIZE).build(); + blobInfo = blobInfo.toBuilder().blobId(BlobId.of("b", "n", GENERATION)).size(SIZE).build(); compareBlobs(BLOB_INFO, blobInfo); } @@ -150,6 +149,6 @@ public void testToPbAndFromPb() { @Test public void testBlobId() { - assertEquals(BlobId.of("b", "n"), BLOB_INFO.blobId()); + assertEquals(BlobId.of("b", "n", GENERATION), BLOB_INFO.blobId()); } } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index 423e972a8de6..a8fb0365e598 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -129,8 +129,8 @@ public void testCreateBlob() { BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); BlobInfo remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT); assertNotNull(remoteBlob); - assertEquals(blob.blobId(), remoteBlob.blobId()); - byte[] readBytes = storage.readAllBytes(BUCKET, blobName); + assertEquals(blob.bucket(), remoteBlob.bucket()); + assertEquals(blob.name(), remoteBlob.name()); byte[] readBytes = storage.readAllBytes(BUCKET, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); assertTrue(storage.delete(BUCKET, blobName)); } @@ -141,7 +141,8 @@ public void testCreateEmptyBlob() { BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); BlobInfo remoteBlob = storage.create(blob); assertNotNull(remoteBlob); - assertEquals(blob.blobId(), remoteBlob.blobId()); + assertEquals(blob.bucket(), remoteBlob.bucket()); + assertEquals(blob.name(), remoteBlob.name()); byte[] readBytes = storage.readAllBytes(BUCKET, blobName); assertArrayEquals(new byte[0], readBytes); assertTrue(storage.delete(BUCKET, blobName)); @@ -154,7 +155,8 @@ public void testCreateBlobStream() throws UnsupportedEncodingException { ByteArrayInputStream stream = new ByteArrayInputStream(BLOB_STRING_CONTENT.getBytes(UTF_8)); BlobInfo remoteBlob = storage.create(blob, stream); assertNotNull(remoteBlob); - assertEquals(blob.blobId(), remoteBlob.blobId()); + assertEquals(blob.bucket(), remoteBlob.bucket()); + assertEquals(blob.name(), remoteBlob.name()); assertEquals(blob.contentType(), remoteBlob.contentType()); byte[] readBytes = storage.readAllBytes(BUCKET, blobName); assertEquals(BLOB_STRING_CONTENT, new String(readBytes, UTF_8)); @@ -166,8 +168,9 @@ public void testCreateBlobFail() { String blobName = "test-create-blob-fail"; BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); assertNotNull(storage.create(blob)); + BlobInfo wrongGenerationBlob = BlobInfo.builder(BUCKET, blobName, -1L).build(); try { - storage.create(blob.toBuilder().generation(-1L).build(), BLOB_BYTE_CONTENT, + storage.create(wrongGenerationBlob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.generationMatch()); fail("StorageException was expected"); } catch (StorageException ex) { @@ -229,13 +232,29 @@ public void testGetBlobAllSelectedFields() { assertNotNull(storage.create(blob)); BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields(BlobField.values())); - assertEquals(blob.blobId(), remoteBlob.blobId()); + assertEquals(blob.bucket(), remoteBlob.bucket()); + assertEquals(blob.name(), remoteBlob.name()); assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata()); assertNotNull(remoteBlob.id()); assertNotNull(remoteBlob.selfLink()); assertTrue(storage.delete(BUCKET, blobName)); } + @Test + public void testGetBlobFail() { + String blobName = "test-get-blob-fail"; + BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); + assertNotNull(storage.create(blob)); + BlobId wrongGenerationBlob = BlobId.of(BUCKET, blobName, -1L); + try { + storage.get(wrongGenerationBlob, Storage.BlobGetOption.generationMatch()); + fail("StorageException was expected"); + } catch (StorageException ex) { + // expected + } + assertTrue(storage.delete(BUCKET, blobName)); + } + @Test public void testListBlobsSelectedFields() { String[] blobNames = {"test-list-blobs-selected-fields-blob1", @@ -297,7 +316,8 @@ public void testUpdateBlob() { assertNotNull(storage.create(blob)); BlobInfo updatedBlob = storage.update(blob.toBuilder().contentType(CONTENT_TYPE).build()); assertNotNull(updatedBlob); - assertEquals(blob.blobId(), updatedBlob.blobId()); + assertEquals(blob.name(), updatedBlob.name()); + assertEquals(blob.bucket(), updatedBlob.bucket()); assertEquals(CONTENT_TYPE, updatedBlob.contentType()); assertTrue(storage.delete(BUCKET, blobName)); } @@ -316,7 +336,8 @@ public void testUpdateBlobReplaceMetadata() { assertNotNull(updatedBlob); assertNull(updatedBlob.metadata()); updatedBlob = storage.update(blob.toBuilder().metadata(newMetadata).build()); - assertEquals(blob.blobId(), updatedBlob.blobId()); + assertEquals(blob.name(), updatedBlob.name()); + assertEquals(blob.bucket(), updatedBlob.bucket()); assertEquals(newMetadata, updatedBlob.metadata()); assertTrue(storage.delete(BUCKET, blobName)); } @@ -334,7 +355,8 @@ public void testUpdateBlobMergeMetadata() { assertNotNull(storage.create(blob)); BlobInfo updatedBlob = storage.update(blob.toBuilder().metadata(newMetadata).build()); assertNotNull(updatedBlob); - assertEquals(blob.blobId(), updatedBlob.blobId()); + assertEquals(blob.name(), updatedBlob.name()); + assertEquals(blob.bucket(), updatedBlob.bucket()); assertEquals(expectedMetadata, updatedBlob.metadata()); assertTrue(storage.delete(BUCKET, blobName)); } @@ -354,7 +376,8 @@ public void testUpdateBlobUnsetMetadata() { assertNotNull(storage.create(blob)); BlobInfo updatedBlob = storage.update(blob.toBuilder().metadata(newMetadata).build()); assertNotNull(updatedBlob); - assertEquals(blob.blobId(), updatedBlob.blobId()); + assertEquals(blob.name(), updatedBlob.name()); + assertEquals(blob.bucket(), updatedBlob.bucket()); assertEquals(expectedMetadata, updatedBlob.metadata()); assertTrue(storage.delete(BUCKET, blobName)); } @@ -364,9 +387,11 @@ public void testUpdateBlobFail() { String blobName = "test-update-blob-fail"; BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); assertNotNull(storage.create(blob)); + BlobInfo wrongGenerationBlob = BlobInfo.builder(BUCKET, blobName, -1L) + .contentType(CONTENT_TYPE) + .build(); try { - storage.update(blob.toBuilder().contentType(CONTENT_TYPE).generation(-1L).build(), - Storage.BlobTargetOption.generationMatch()); + storage.update(wrongGenerationBlob, Storage.BlobTargetOption.generationMatch()); fail("StorageException was expected"); } catch (StorageException ex) { // expected @@ -408,7 +433,8 @@ public void testComposeBlob() { Storage.ComposeRequest.of(ImmutableList.of(sourceBlobName1, sourceBlobName2), targetBlob); BlobInfo remoteBlob = storage.compose(req); assertNotNull(remoteBlob); - assertEquals(targetBlob.blobId(), remoteBlob.blobId()); + assertEquals(targetBlob.name(), remoteBlob.name()); + assertEquals(targetBlob.bucket(), remoteBlob.bucket()); byte[] readBytes = storage.readAllBytes(BUCKET, targetBlobName); byte[] composedBytes = Arrays.copyOf(BLOB_BYTE_CONTENT, BLOB_BYTE_CONTENT.length * 2); System.arraycopy(BLOB_BYTE_CONTENT, 0, composedBytes, BLOB_BYTE_CONTENT.length, @@ -491,12 +517,12 @@ public void testCopyBlobUpdateMetadata() { @Test public void testCopyBlobFail() { String sourceBlobName = "test-copy-blob-source-fail"; - BlobId source = BlobId.of(BUCKET, sourceBlobName); + BlobId source = BlobId.of(BUCKET, sourceBlobName, -1L); assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); String targetBlobName = "test-copy-blob-target-fail"; BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build(); Storage.CopyRequest req = Storage.CopyRequest.builder() - .source(source) + .source(BUCKET, sourceBlobName) .sourceOptions(Storage.BlobSourceOption.generationMatch(-1L)) .target(target) .build(); @@ -506,6 +532,17 @@ public void testCopyBlobFail() { } catch (StorageException ex) { // expected } + Storage.CopyRequest req2 = Storage.CopyRequest.builder() + .source(source) + .sourceOptions(Storage.BlobSourceOption.generationMatch()) + .target(target) + .build(); + try { + storage.copy(req2); + fail("StorageException was expected"); + } catch (StorageException ex) { + // expected + } assertTrue(storage.delete(BUCKET, sourceBlobName)); } @@ -531,8 +568,10 @@ public void testBatchRequest() { assertEquals(0, updateResponse.gets().size()); BlobInfo remoteUpdatedBlob1 = updateResponse.updates().get(0).get(); BlobInfo remoteUpdatedBlob2 = updateResponse.updates().get(1).get(); - assertEquals(sourceBlob1.blobId(), remoteUpdatedBlob1.blobId()); - assertEquals(sourceBlob2.blobId(), remoteUpdatedBlob2.blobId()); + assertEquals(sourceBlob1.bucket(), remoteUpdatedBlob1.bucket()); + assertEquals(sourceBlob1.name(), remoteUpdatedBlob1.name()); + assertEquals(sourceBlob2.bucket(), remoteUpdatedBlob2.bucket()); + assertEquals(sourceBlob2.name(), remoteUpdatedBlob2.name()); assertEquals(updatedBlob1.contentType(), remoteUpdatedBlob1.contentType()); assertEquals(updatedBlob2.contentType(), remoteUpdatedBlob2.contentType()); @@ -568,19 +607,23 @@ public void testBatchRequestFail() { String blobName = "test-batch-request-blob-fail"; BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); assertNotNull(storage.create(blob)); - BlobInfo updatedBlob = blob.toBuilder().generation(-1L).build(); + BlobInfo updatedBlob = BlobInfo.builder(BUCKET, blobName, -1L).build(); BatchRequest batchRequest = BatchRequest.builder() .update(updatedBlob, Storage.BlobTargetOption.generationMatch()) .delete(BUCKET, blobName, Storage.BlobSourceOption.generationMatch(-1L)) + .delete(BlobId.of(BUCKET, blobName, -1L), Storage.BlobSourceOption.generationMatch()) .get(BUCKET, blobName, Storage.BlobGetOption.generationMatch(-1L)) + .get(BlobId.of(BUCKET, blobName, -1L), Storage.BlobGetOption.generationMatch()) .build(); BatchResponse updateResponse = storage.apply(batchRequest); assertEquals(1, updateResponse.updates().size()); - assertEquals(1, updateResponse.deletes().size()); - assertEquals(1, updateResponse.gets().size()); + assertEquals(2, updateResponse.deletes().size()); + assertEquals(2, updateResponse.gets().size()); assertTrue(updateResponse.updates().get(0).failed()); assertTrue(updateResponse.gets().get(0).failed()); + assertTrue(updateResponse.gets().get(1).failed()); assertTrue(updateResponse.deletes().get(0).failed()); + assertTrue(updateResponse.deletes().get(1).failed()); assertTrue(storage.delete(BUCKET, blobName)); } @@ -648,13 +691,28 @@ public void testReadChannelFail() throws IOException { } catch (StorageException ex) { // expected } + try (BlobReadChannel reader = + storage.reader(blob.blobId(), Storage.BlobSourceOption.generationMatch(-1L))) { + reader.read(ByteBuffer.allocate(42)); + fail("StorageException was expected"); + } catch (StorageException ex) { + // expected + } + BlobId blobIdWrongGeneration = BlobId.of(BUCKET, blobName, -1L); + try (BlobReadChannel reader = + storage.reader(blobIdWrongGeneration, Storage.BlobSourceOption.generationMatch())) { + reader.read(ByteBuffer.allocate(42)); + fail("StorageException was expected"); + } catch (StorageException ex) { + // expected + } assertTrue(storage.delete(BUCKET, blobName)); } @Test public void testWriteChannelFail() throws IOException { String blobName = "test-write-channel-blob-fail"; - BlobInfo blob = BlobInfo.builder(BUCKET, blobName).generation(-1L).build(); + BlobInfo blob = BlobInfo.builder(BUCKET, blobName, -1L).build(); try { try (BlobWriteChannel writer = storage.writer(blob, Storage.BlobWriteOption.generationMatch())) { @@ -707,7 +765,8 @@ public void testPostSignedUrl() throws IOException { connection.connect(); BlobInfo remoteBlob = storage.get(BUCKET, blobName); assertNotNull(remoteBlob); - assertEquals(blob.blobId(), remoteBlob.blobId()); + assertEquals(blob.bucket(), remoteBlob.bucket()); + assertEquals(blob.name(), remoteBlob.name()); assertTrue(storage.delete(BUCKET, blobName)); } @@ -720,8 +779,10 @@ public void testGetBlobs() { assertNotNull(storage.create(sourceBlob1)); assertNotNull(storage.create(sourceBlob2)); List remoteBlobs = storage.get(sourceBlob1.blobId(), sourceBlob2.blobId()); - assertEquals(sourceBlob1.blobId(), remoteBlobs.get(0).blobId()); - assertEquals(sourceBlob2.blobId(), remoteBlobs.get(1).blobId()); + assertEquals(sourceBlob1.bucket(), remoteBlobs.get(0).bucket()); + assertEquals(sourceBlob1.name(), remoteBlobs.get(0).name()); + assertEquals(sourceBlob2.bucket(), remoteBlobs.get(1).bucket()); + assertEquals(sourceBlob2.name(), remoteBlobs.get(1).name()); assertTrue(storage.delete(BUCKET, sourceBlobName1)); assertTrue(storage.delete(BUCKET, sourceBlobName2)); } @@ -734,7 +795,8 @@ public void testGetBlobsFail() { BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build(); assertNotNull(storage.create(sourceBlob1)); List remoteBlobs = storage.get(sourceBlob1.blobId(), sourceBlob2.blobId()); - assertEquals(sourceBlob1.blobId(), remoteBlobs.get(0).blobId()); + assertEquals(sourceBlob1.bucket(), remoteBlobs.get(0).bucket()); + assertEquals(sourceBlob1.name(), remoteBlobs.get(0).name()); assertNull(remoteBlobs.get(1)); assertTrue(storage.delete(BUCKET, sourceBlobName1)); } @@ -777,9 +839,11 @@ public void testUpdateBlobs() { List updatedBlobs = storage.update( remoteBlob1.toBuilder().contentType(CONTENT_TYPE).build(), remoteBlob2.toBuilder().contentType(CONTENT_TYPE).build()); - assertEquals(sourceBlob1.blobId(), updatedBlobs.get(0).blobId()); + assertEquals(sourceBlob1.bucket(), updatedBlobs.get(0).bucket()); + assertEquals(sourceBlob1.name(), updatedBlobs.get(0).name()); assertEquals(CONTENT_TYPE, updatedBlobs.get(0).contentType()); - assertEquals(sourceBlob2.blobId(), updatedBlobs.get(1).blobId()); + assertEquals(sourceBlob2.bucket(), updatedBlobs.get(1).bucket()); + assertEquals(sourceBlob2.name(), updatedBlobs.get(1).name()); assertEquals(CONTENT_TYPE, updatedBlobs.get(1).contentType()); assertTrue(storage.delete(BUCKET, sourceBlobName1)); assertTrue(storage.delete(BUCKET, sourceBlobName2)); @@ -796,7 +860,8 @@ public void testUpdateBlobsFail() { List updatedBlobs = storage.update( remoteBlob1.toBuilder().contentType(CONTENT_TYPE).build(), sourceBlob2.toBuilder().contentType(CONTENT_TYPE).build()); - assertEquals(sourceBlob1.blobId(), updatedBlobs.get(0).blobId()); + assertEquals(sourceBlob1.bucket(), updatedBlobs.get(0).bucket()); + assertEquals(sourceBlob1.name(), updatedBlobs.get(0).name()); assertEquals(CONTENT_TYPE, updatedBlobs.get(0).contentType()); assertNull(updatedBlobs.get(1)); assertTrue(storage.delete(BUCKET, sourceBlobName1)); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java index f07c7000813e..849e2ed14499 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java @@ -89,8 +89,8 @@ public class StorageImplTest { private static final BucketInfo BUCKET_INFO2 = BucketInfo.builder(BUCKET_NAME2).build(); // BlobInfo objects - private static final BlobInfo BLOB_INFO1 = BlobInfo.builder(BUCKET_NAME1, BLOB_NAME1) - .metageneration(42L).generation(24L).contentType("application/json").md5("md5string").build(); + private static final BlobInfo BLOB_INFO1 = BlobInfo.builder(BUCKET_NAME1, BLOB_NAME1, 24L) + .metageneration(42L).contentType("application/json").md5("md5string").build(); private static final BlobInfo BLOB_INFO2 = BlobInfo.builder(BUCKET_NAME1, BLOB_NAME2).build(); private static final BlobInfo BLOB_INFO3 = BlobInfo.builder(BUCKET_NAME1, BLOB_NAME3).build(); @@ -157,6 +157,8 @@ public class StorageImplTest { Storage.BlobGetOption.metagenerationMatch(BLOB_INFO1.metageneration()); private static final Storage.BlobGetOption BLOB_GET_GENERATION = Storage.BlobGetOption.generationMatch(BLOB_INFO1.generation()); + private static final Storage.BlobGetOption BLOB_GET_GENERATION_FROM_BLOB_ID = + Storage.BlobGetOption.generationMatch(); private static final Storage.BlobGetOption BLOB_GET_FIELDS = Storage.BlobGetOption.fields(Storage.BlobField.CONTENT_TYPE, Storage.BlobField.CRC32C); private static final Storage.BlobGetOption BLOB_GET_EMPTY_FIELDS = @@ -168,6 +170,8 @@ public class StorageImplTest { Storage.BlobSourceOption.metagenerationMatch(BLOB_INFO1.metageneration()); private static final Storage.BlobSourceOption BLOB_SOURCE_GENERATION = Storage.BlobSourceOption.generationMatch(BLOB_INFO1.generation()); + private static final Storage.BlobSourceOption BLOB_SOURCE_GENERATION_FROM_BLOB_ID = + Storage.BlobSourceOption.generationMatch(); private static final Map BLOB_SOURCE_OPTIONS = ImmutableMap.of( StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_SOURCE_METAGENERATION.value(), StorageRpc.Option.IF_GENERATION_MATCH, BLOB_SOURCE_GENERATION.value()); @@ -454,6 +458,18 @@ public void testGetBlobWithOptions() { assertEquals(BLOB_INFO1, blob); } + @Test + public void testGetBlobWithOptionsFromBlobId() { + EasyMock.expect( + storageRpcMock.get(BLOB_INFO1.blobId().toPb(), BLOB_GET_OPTIONS)) + .andReturn(BLOB_INFO1.toPb()); + EasyMock.replay(storageRpcMock); + storage = options.service(); + BlobInfo blob = + storage.get(BLOB_INFO1.blobId(), BLOB_GET_METAGENERATION, BLOB_GET_GENERATION_FROM_BLOB_ID); + assertEquals(BLOB_INFO1, blob); + } + @Test public void testGetBlobWithSelectedFields() { Capture> capturedOptions = @@ -766,6 +782,17 @@ public void testDeleteBlobWithOptions() { BLOB_SOURCE_METAGENERATION)); } + @Test + public void testDeleteBlobWithOptionsFromBlobId() { + EasyMock.expect( + storageRpcMock.delete(BLOB_INFO1.blobId().toPb(), BLOB_SOURCE_OPTIONS)) + .andReturn(true); + EasyMock.replay(storageRpcMock); + storage = options.service(); + assertTrue(storage.delete(BLOB_INFO1.blobId(), BLOB_SOURCE_GENERATION_FROM_BLOB_ID, + BLOB_SOURCE_METAGENERATION)); + } + @Test public void testCompose() { Storage.ComposeRequest req = Storage.ComposeRequest.builder() @@ -831,6 +858,26 @@ public void testCopyWithOptions() { assertTrue(!writer.isDone()); } + @Test + public void testCopyWithOptionsFromBlobId() { + CopyRequest request = Storage.CopyRequest.builder() + .source(BLOB_INFO1.blobId()) + .sourceOptions(BLOB_SOURCE_GENERATION_FROM_BLOB_ID, BLOB_SOURCE_METAGENERATION) + .target(BLOB_INFO1, BLOB_TARGET_GENERATION, BLOB_TARGET_METAGENERATION) + .build(); + StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), + BLOB_SOURCE_OPTIONS_COPY, request.target().toPb(), BLOB_TARGET_OPTIONS_COMPOSE, null); + StorageRpc.RewriteResponse rpcResponse = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, + false, "token", 21L); + EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse); + EasyMock.replay(storageRpcMock); + storage = options.service(); + CopyWriter writer = storage.copy(request); + assertEquals(42L, writer.blobSize()); + assertEquals(21L, writer.totalBytesCopied()); + assertTrue(!writer.isDone()); + } + @Test public void testCopyMultipleRequests() { CopyRequest request = Storage.CopyRequest.of(BLOB_INFO1.blobId(), BLOB_INFO2.blobId()); @@ -877,6 +924,18 @@ public void testReadAllBytesWithOptions() { assertArrayEquals(BLOB_CONTENT, readBytes); } + @Test + public void testReadAllBytesWithOptionsFromBlobId() { + EasyMock.expect( + storageRpcMock.load(BLOB_INFO1.blobId().toPb(), BLOB_SOURCE_OPTIONS)) + .andReturn(BLOB_CONTENT); + EasyMock.replay(storageRpcMock); + storage = options.service(); + byte[] readBytes = storage.readAllBytes(BLOB_INFO1.blobId(), + BLOB_SOURCE_GENERATION_FROM_BLOB_ID, BLOB_SOURCE_METAGENERATION); + assertArrayEquals(BLOB_CONTENT, readBytes); + } + @Test public void testApply() { BatchRequest req = BatchRequest.builder() @@ -981,6 +1040,21 @@ public void testReaderWithOptions() throws IOException { channel.read(ByteBuffer.allocate(42)); } + @Test + public void testReaderWithOptionsFromBlobId() throws IOException { + byte[] result = new byte[DEFAULT_CHUNK_SIZE]; + EasyMock.expect( + storageRpcMock.read(BLOB_INFO1.blobId().toPb(), BLOB_SOURCE_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) + .andReturn(result); + EasyMock.replay(storageRpcMock); + storage = options.service(); + BlobReadChannel channel = storage.reader(BLOB_INFO1.blobId(), + BLOB_SOURCE_GENERATION_FROM_BLOB_ID, BLOB_SOURCE_METAGENERATION); + assertNotNull(channel); + assertTrue(channel.isOpen()); + channel.read(ByteBuffer.allocate(42)); + } + @Test public void testWriter() { BlobInfo.Builder infoBuilder = BLOB_INFO1.toBuilder(); From 6c961b624e8c3612f761b288a0c38b52ce43f97d Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Thu, 12 Nov 2015 15:17:53 -0800 Subject: [PATCH 37/94] Create version 0.0.12 --- gcloud-java-core/pom.xml | 2 +- gcloud-java-datastore/pom.xml | 2 +- gcloud-java-examples/pom.xml | 2 +- gcloud-java-storage/pom.xml | 2 +- gcloud-java/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml index e13933bd2beb..c89347130005 100644 --- a/gcloud-java-core/pom.xml +++ b/gcloud-java-core/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.12-SNAPSHOT + 0.0.12 gcloud-java-core diff --git a/gcloud-java-datastore/pom.xml b/gcloud-java-datastore/pom.xml index b58e9e0ffc74..2e9f48970a68 100644 --- a/gcloud-java-datastore/pom.xml +++ b/gcloud-java-datastore/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.12-SNAPSHOT + 0.0.12 gcloud-java-datastore diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml index c461846acab2..feac050f49ae 100644 --- a/gcloud-java-examples/pom.xml +++ b/gcloud-java-examples/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.12-SNAPSHOT + 0.0.12 gcloud-java-examples diff --git a/gcloud-java-storage/pom.xml b/gcloud-java-storage/pom.xml index ef3ddec79816..bddd11a22037 100644 --- a/gcloud-java-storage/pom.xml +++ b/gcloud-java-storage/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.12-SNAPSHOT + 0.0.12 gcloud-java-storage diff --git a/gcloud-java/pom.xml b/gcloud-java/pom.xml index d26c38f517d5..e44704973908 100644 --- a/gcloud-java/pom.xml +++ b/gcloud-java/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.12-SNAPSHOT + 0.0.12 diff --git a/pom.xml b/pom.xml index 78cbfe11d351..32a73e175b70 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.gcloud gcloud-java-pom pom - 0.0.12-SNAPSHOT + 0.0.12 GCloud Java https://github.com/GoogleCloudPlatform/gcloud-java From 2fd8066e891fb3dfea69b65f6bf6461db79342b9 Mon Sep 17 00:00:00 2001 From: travis-ci Date: Thu, 12 Nov 2015 23:43:27 +0000 Subject: [PATCH 38/94] Updating version in README files. --- README.md | 6 +++--- gcloud-java-core/README.md | 6 +++--- gcloud-java-datastore/README.md | 6 +++--- gcloud-java-examples/README.md | 6 +++--- gcloud-java-storage/README.md | 6 +++--- gcloud-java/README.md | 6 +++--- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5ed44e91adfd..bba06a7e60d4 100644 --- a/README.md +++ b/README.md @@ -25,16 +25,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java - 0.0.11 + 0.0.12 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java:jar:0.0.11' +compile 'com.google.gcloud:gcloud-java:jar:0.0.12' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.11" +libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.12" ``` Example Applications diff --git a/gcloud-java-core/README.md b/gcloud-java-core/README.md index f84fb33993e7..714c43f5c63d 100644 --- a/gcloud-java-core/README.md +++ b/gcloud-java-core/README.md @@ -17,16 +17,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java-core - 0.0.11 + 0.0.12 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java-core:jar:0.0.11' +compile 'com.google.gcloud:gcloud-java-core:jar:0.0.12' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java-core" % "0.0.11" +libraryDependencies += "com.google.gcloud" % "gcloud-java-core" % "0.0.12" ``` Java Versions diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md index 6d9fc0e8c4d6..2525413ecb86 100644 --- a/gcloud-java-datastore/README.md +++ b/gcloud-java-datastore/README.md @@ -20,16 +20,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java-datastore - 0.0.11 + 0.0.12 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java-datastore:jar:0.0.11' +compile 'com.google.gcloud:gcloud-java-datastore:jar:0.0.12' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java-datastore" % "0.0.11" +libraryDependencies += "com.google.gcloud" % "gcloud-java-datastore" % "0.0.12" ``` Example Application diff --git a/gcloud-java-examples/README.md b/gcloud-java-examples/README.md index bc738de41b51..f381465676f5 100644 --- a/gcloud-java-examples/README.md +++ b/gcloud-java-examples/README.md @@ -17,16 +17,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java-examples - 0.0.11 + 0.0.12 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java-examples:jar:0.0.11' +compile 'com.google.gcloud:gcloud-java-examples:jar:0.0.12' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java-examples" % "0.0.11" +libraryDependencies += "com.google.gcloud" % "gcloud-java-examples" % "0.0.12" ``` To run examples from your command line: diff --git a/gcloud-java-storage/README.md b/gcloud-java-storage/README.md index f2b99388ff0f..8b19484dd14d 100644 --- a/gcloud-java-storage/README.md +++ b/gcloud-java-storage/README.md @@ -20,16 +20,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java-storage - 0.0.11 + 0.0.12 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java-storage:jar:0.0.11' +compile 'com.google.gcloud:gcloud-java-storage:jar:0.0.12' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java-storage" % "0.0.11" +libraryDependencies += "com.google.gcloud" % "gcloud-java-storage" % "0.0.12" ``` Example Application diff --git a/gcloud-java/README.md b/gcloud-java/README.md index eaaed21af5fe..9a4ddfe77c83 100644 --- a/gcloud-java/README.md +++ b/gcloud-java/README.md @@ -25,16 +25,16 @@ If you are using Maven, add this to your pom.xml file com.google.gcloud gcloud-java - 0.0.11 + 0.0.12 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.gcloud:gcloud-java:jar:0.0.11' +compile 'com.google.gcloud:gcloud-java:jar:0.0.12' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.11" +libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.12" ``` Java Versions From af10dbfbc47857d51ae581cbdd583ea98bbbc898 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Thu, 12 Nov 2015 16:10:30 -0800 Subject: [PATCH 39/94] Update version to 0.0.13-SNAPSHOT --- gcloud-java-core/pom.xml | 2 +- gcloud-java-datastore/pom.xml | 2 +- gcloud-java-examples/pom.xml | 2 +- gcloud-java-storage/pom.xml | 2 +- gcloud-java/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml index c89347130005..7e184db2dbfa 100644 --- a/gcloud-java-core/pom.xml +++ b/gcloud-java-core/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.12 + 0.0.13-SNAPSHOT gcloud-java-core diff --git a/gcloud-java-datastore/pom.xml b/gcloud-java-datastore/pom.xml index 2e9f48970a68..f2743c8e2af1 100644 --- a/gcloud-java-datastore/pom.xml +++ b/gcloud-java-datastore/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.12 + 0.0.13-SNAPSHOT gcloud-java-datastore diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml index feac050f49ae..2b02be47cc40 100644 --- a/gcloud-java-examples/pom.xml +++ b/gcloud-java-examples/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.12 + 0.0.13-SNAPSHOT gcloud-java-examples diff --git a/gcloud-java-storage/pom.xml b/gcloud-java-storage/pom.xml index bddd11a22037..b592200243ef 100644 --- a/gcloud-java-storage/pom.xml +++ b/gcloud-java-storage/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.12 + 0.0.13-SNAPSHOT gcloud-java-storage diff --git a/gcloud-java/pom.xml b/gcloud-java/pom.xml index e44704973908..214094dcd104 100644 --- a/gcloud-java/pom.xml +++ b/gcloud-java/pom.xml @@ -11,7 +11,7 @@ com.google.gcloud gcloud-java-pom - 0.0.12 + 0.0.13-SNAPSHOT diff --git a/pom.xml b/pom.xml index 32a73e175b70..234290a6da44 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.gcloud gcloud-java-pom pom - 0.0.12 + 0.0.13-SNAPSHOT GCloud Java https://github.com/GoogleCloudPlatform/gcloud-java From f0ce8972e1a414a620de6d9c83f82cbce7520264 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 13 Nov 2015 10:32:11 +0100 Subject: [PATCH 40/94] Remove Blob(Source/Get)Option.setGeneration method, add Storage.optionMap --- .../google/gcloud/storage/BatchRequest.java | 4 +- .../com/google/gcloud/storage/Storage.java | 39 +------------------ .../google/gcloud/storage/StorageImpl.java | 22 +++++------ .../gcloud/storage/BatchRequestTest.java | 4 +- .../gcloud/storage/StorageImplTest.java | 4 +- 5 files changed, 18 insertions(+), 55 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java index ae48c5790a77..bf77c731754e 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java @@ -58,7 +58,7 @@ public Builder delete(String bucket, String blob, BlobSourceOption... options) { * Delete the given blob. */ public Builder delete(BlobId blob, BlobSourceOption... options) { - toDelete.put(blob, Lists.newArrayList(BlobSourceOption.setGeneration(blob, options))); + toDelete.put(blob, Lists.newArrayList(options)); return this; } @@ -82,7 +82,7 @@ public Builder get(String bucket, String blob, BlobGetOption... options) { * Retrieve metadata for the given blob. */ public Builder get(BlobId blob, BlobGetOption... options) { - toGet.put(blob, Lists.newArrayList(BlobGetOption.setGeneration(blob, options))); + toGet.put(blob, Lists.newArrayList(options)); return this; } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 4210a278c413..b61cfb269c1a 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -526,26 +526,6 @@ public static BlobSourceOption metagenerationMatch(long metageneration) { public static BlobSourceOption metagenerationNotMatch(long metageneration) { return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration); } - - static BlobSourceOption[] setGeneration(BlobId blobId, Iterable options) { - return setGeneration(blobId, Iterables.toArray(options, BlobSourceOption.class)); - } - - static BlobSourceOption[] setGeneration(BlobId blobId, BlobSourceOption... options) { - BlobSourceOption[] updatedOptions = new BlobSourceOption[options.length]; - int index = 0; - for (BlobSourceOption option : options) { - if ((option.rpcOption() == StorageRpc.Option.IF_GENERATION_MATCH - || option.rpcOption() == StorageRpc.Option.IF_GENERATION_NOT_MATCH) - && option.value() == null) { - updatedOptions[index] = new BlobSourceOption(option.rpcOption(), blobId.generation()); - } else { - updatedOptions[index] = option; - } - index++; - } - return updatedOptions; - } } /** @@ -622,22 +602,6 @@ public static BlobGetOption metagenerationNotMatch(long metageneration) { public static BlobGetOption fields(BlobField... fields) { return new BlobGetOption(StorageRpc.Option.FIELDS, BlobField.selector(fields)); } - - static BlobGetOption[] setGeneration(BlobId blobId, BlobGetOption... options) { - BlobGetOption[] updatedOptions = new BlobGetOption[options.length]; - int index = 0; - for (BlobGetOption option : options) { - if ((option.rpcOption() == StorageRpc.Option.IF_GENERATION_MATCH - || option.rpcOption() == StorageRpc.Option.IF_GENERATION_NOT_MATCH) - && option.value() == null) { - updatedOptions[index] = new BlobGetOption(option.rpcOption(), blobId.generation()); - } else { - updatedOptions[index] = option; - } - index++; - } - return updatedOptions; - } } /** @@ -1091,8 +1055,7 @@ public CopyRequest build() { private CopyRequest(Builder builder) { source = checkNotNull(builder.source); - sourceOptions = ImmutableList.copyOf( - BlobSourceOption.setGeneration(source, builder.sourceOptions)); + sourceOptions = ImmutableList.copyOf(builder.sourceOptions); target = checkNotNull(builder.target); targetOptions = ImmutableList.copyOf(builder.targetOptions); megabytesCopiedPerChunk = builder.megabytesCopiedPerChunk; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index b4da9e14fd97..fa059254eddb 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -199,8 +199,7 @@ public BlobInfo get(String bucket, String blob, BlobGetOption... options) { @Override public BlobInfo get(BlobId blob, BlobGetOption... options) { final StorageObject storedObject = blob.toPb(); - final Map optionsMap = - optionMap(BlobGetOption.setGeneration(blob, options)); + final Map optionsMap = optionMap(blob, options); try { StorageObject storageObject = runWithRetries(new Callable() { @Override @@ -406,8 +405,7 @@ public boolean delete(String bucket, String blob, BlobSourceOption... options) { @Override public boolean delete(BlobId blob, BlobSourceOption... options) { final StorageObject storageObject = blob.toPb(); - final Map optionsMap = - optionMap(BlobSourceOption.setGeneration(blob, options)); + final Map optionsMap = optionMap(blob, options); try { return runWithRetries(new Callable() { @Override @@ -453,7 +451,7 @@ public StorageObject call() { public CopyWriter copy(final CopyRequest copyRequest) { final StorageObject source = copyRequest.source().toPb(); final Map sourceOptions = - optionMap(null, null, copyRequest.sourceOptions(), true); + optionMap(copyRequest.source().generation(), null, copyRequest.sourceOptions(), true); final StorageObject target = copyRequest.target().toPb(); final Map targetOptions = optionMap(copyRequest.target().generation(), copyRequest.target().metageneration(), copyRequest.targetOptions()); @@ -479,8 +477,7 @@ public byte[] readAllBytes(String bucket, String blob, BlobSourceOption... optio @Override public byte[] readAllBytes(BlobId blob, BlobSourceOption... options) { final StorageObject storageObject = blob.toPb(); - final Map optionsMap = - optionMap(BlobSourceOption.setGeneration(blob, options)); + final Map optionsMap = optionMap(blob, options); try { return runWithRetries(new Callable() { @Override @@ -499,7 +496,7 @@ public BatchResponse apply(BatchRequest batchRequest) { Lists.newArrayListWithCapacity(batchRequest.toDelete().size()); for (Map.Entry> entry : batchRequest.toDelete().entrySet()) { BlobId blob = entry.getKey(); - Map optionsMap = optionMap(null, null, entry.getValue()); + Map optionsMap = optionMap(blob.generation(), null, entry.getValue()); StorageObject storageObject = blob.toPb(); toDelete.add(Tuple.>of(storageObject, optionsMap)); } @@ -516,7 +513,7 @@ public BatchResponse apply(BatchRequest batchRequest) { Lists.newArrayListWithCapacity(batchRequest.toGet().size()); for (Map.Entry> entry : batchRequest.toGet().entrySet()) { BlobId blob = entry.getKey(); - Map optionsMap = optionMap(null, null, entry.getValue()); + Map optionsMap = optionMap(blob.generation(), null, entry.getValue()); toGet.add(Tuple.>of(blob.toPb(), optionsMap)); } StorageRpc.BatchResponse response = @@ -561,8 +558,7 @@ public BlobReadChannel reader(String bucket, String blob, BlobSourceOption... op @Override public BlobReadChannel reader(BlobId blob, BlobSourceOption... options) { - Map optionsMap = - optionMap(BlobSourceOption.setGeneration(blob, options)); + Map optionsMap = optionMap(blob, options); return new BlobReadChannelImpl(options(), blob, optionsMap); } @@ -746,4 +742,8 @@ private static void addToOptionMap(StorageRpc.Option getOption, StorageRpc.O private Map optionMap(BlobInfo blobInfo, Option... options) { return optionMap(blobInfo.generation(), blobInfo.metageneration(), options); } + + private Map optionMap(BlobId blobId, Option... options) { + return optionMap(blobId.generation(), null, options); + } } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java index 020bdf5ed404..63972ff85dfd 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java @@ -52,7 +52,7 @@ public void testBatchRequest() { Entry> delete = deletes.next(); assertEquals(BlobId.of("b1", "o1", 1L), delete.getKey()); assertEquals(1, Iterables.size(delete.getValue())); - assertEquals(BlobSourceOption.generationMatch(1L), Iterables.getFirst(delete.getValue(), null)); + assertEquals(BlobSourceOption.generationMatch(), Iterables.getFirst(delete.getValue(), null)); delete = deletes.next(); assertEquals(BlobId.of("b1", "o2"), delete.getKey()); assertEquals(2, Iterables.size(delete.getValue())); @@ -77,7 +77,7 @@ public void testBatchRequest() { Entry> get = gets.next(); assertEquals(BlobId.of("b3", "o1", 1L), get.getKey()); assertEquals(1, Iterables.size(get.getValue())); - assertEquals(BlobGetOption.generationMatch(1), Iterables.getFirst(get.getValue(), null)); + assertEquals(BlobGetOption.generationMatch(), Iterables.getFirst(get.getValue(), null)); get = gets.next(); assertEquals(BlobId.of("b3", "o2"), get.getKey()); assertEquals(1, Iterables.size(get.getValue())); diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java index 849e2ed14499..359920314e82 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java @@ -867,8 +867,8 @@ public void testCopyWithOptionsFromBlobId() { .build(); StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), BLOB_SOURCE_OPTIONS_COPY, request.target().toPb(), BLOB_TARGET_OPTIONS_COMPOSE, null); - StorageRpc.RewriteResponse rpcResponse = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, - false, "token", 21L); + StorageRpc.RewriteResponse rpcResponse = + new StorageRpc.RewriteResponse(rpcRequest, null, 42L, false, "token", 21L); EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse); EasyMock.replay(storageRpcMock); storage = options.service(); From 33119695a9cc05bc7fa0c46fdc1291b2c1a8baf4 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 13 Nov 2015 10:33:20 +0100 Subject: [PATCH 41/94] Set blob generation before sending requests in DefaultStorageRpc --- .../google/gcloud/spi/DefaultStorageRpc.java | 10 +++- .../google/gcloud/storage/ITStorageTest.java | 49 +++++++++++++------ 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index b1e188f1d1fb..d215d6b40e61 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -207,6 +207,7 @@ private Storage.Objects.Get getRequest(StorageObject object, Map opti throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) + .setGeneration(object.getGeneration()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) @@ -288,6 +289,7 @@ private Storage.Objects.Delete deleteRequest(StorageObject blob, Map throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) + .setGeneration(blob.getGeneration()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) @@ -332,6 +334,7 @@ public byte[] load(StorageObject from, Map options) try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) + .setGeneration(from.getGeneration()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) @@ -409,8 +412,10 @@ public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { public byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageException { try { - Get req = storage.objects().get(from.getBucket(), from.getName()); - req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) + Get req = storage.objects() + .get(from.getBucket(), from.getName()) + .setGeneration(from.getGeneration()) + .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); @@ -521,6 +526,7 @@ private RewriteResponse rewrite(RewriteRequest req, String token) throws Storage com.google.api.services.storage.model.RewriteResponse rewriteReponse = storage.objects() .rewrite(req.source.getBucket(), req.source.getName(), req.target.getBucket(), req.target.getName(), req.target.getContentType() != null ? req.target : null) + .setSourceGeneration(req.source.getGeneration()) .setRewriteToken(token) .setMaxBytesRewrittenPerCall(maxBytesRewrittenPerCall) .setProjection(DEFAULT_PROJECTION) diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index a8fb0365e598..d22bf06c8b16 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -19,6 +19,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -130,7 +131,8 @@ public void testCreateBlob() { BlobInfo remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT); assertNotNull(remoteBlob); assertEquals(blob.bucket(), remoteBlob.bucket()); - assertEquals(blob.name(), remoteBlob.name()); byte[] readBytes = storage.readAllBytes(BUCKET, blobName); + assertEquals(blob.name(), remoteBlob.name()); + byte[] readBytes = storage.readAllBytes(BUCKET, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); assertTrue(storage.delete(BUCKET, blobName)); } @@ -245,9 +247,9 @@ public void testGetBlobFail() { String blobName = "test-get-blob-fail"; BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); assertNotNull(storage.create(blob)); - BlobId wrongGenerationBlob = BlobId.of(BUCKET, blobName, -1L); + BlobId wrongGenerationBlob = BlobId.of(BUCKET, blobName); try { - storage.get(wrongGenerationBlob, Storage.BlobGetOption.generationMatch()); + storage.get(wrongGenerationBlob, Storage.BlobGetOption.generationMatch(-1)); fail("StorageException was expected"); } catch (StorageException ex) { // expected @@ -255,6 +257,16 @@ public void testGetBlobFail() { assertTrue(storage.delete(BUCKET, blobName)); } + @Test + public void testGetBlobFailNonExistingGeneration() { + String blobName = "test-get-blob-fail-non-existing-generation"; + BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); + assertNotNull(storage.create(blob)); + BlobId wrongGenerationBlob = BlobId.of(BUCKET, blobName, -1L); + assertNull(storage.get(wrongGenerationBlob)); + assertTrue(storage.delete(BUCKET, blobName)); + } + @Test public void testListBlobsSelectedFields() { String[] blobNames = {"test-list-blobs-selected-fields-blob1", @@ -405,6 +417,14 @@ public void testDeleteNonExistingBlob() { assertTrue(!storage.delete(BUCKET, blobName)); } + @Test + public void testDeleteBlobNonExistingGeneration() { + String blobName = "test-delete-blob-non-existing-generation"; + BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build(); + assertNotNull(storage.create(blob)); + assertTrue(!storage.delete(BlobId.of(BUCKET, blobName, -1L))); + } + @Test public void testDeleteBlobFail() { String blobName = "test-delete-blob-fail"; @@ -611,19 +631,20 @@ public void testBatchRequestFail() { BatchRequest batchRequest = BatchRequest.builder() .update(updatedBlob, Storage.BlobTargetOption.generationMatch()) .delete(BUCKET, blobName, Storage.BlobSourceOption.generationMatch(-1L)) - .delete(BlobId.of(BUCKET, blobName, -1L), Storage.BlobSourceOption.generationMatch()) + .delete(BlobId.of(BUCKET, blobName, -1L)) .get(BUCKET, blobName, Storage.BlobGetOption.generationMatch(-1L)) - .get(BlobId.of(BUCKET, blobName, -1L), Storage.BlobGetOption.generationMatch()) + .get(BlobId.of(BUCKET, blobName, -1L)) .build(); - BatchResponse updateResponse = storage.apply(batchRequest); - assertEquals(1, updateResponse.updates().size()); - assertEquals(2, updateResponse.deletes().size()); - assertEquals(2, updateResponse.gets().size()); - assertTrue(updateResponse.updates().get(0).failed()); - assertTrue(updateResponse.gets().get(0).failed()); - assertTrue(updateResponse.gets().get(1).failed()); - assertTrue(updateResponse.deletes().get(0).failed()); - assertTrue(updateResponse.deletes().get(1).failed()); + BatchResponse batchResponse = storage.apply(batchRequest); + assertEquals(1, batchResponse.updates().size()); + assertEquals(2, batchResponse.deletes().size()); + assertEquals(2, batchResponse.gets().size()); + assertTrue(batchResponse.updates().get(0).failed()); + assertTrue(batchResponse.gets().get(0).failed()); + assertFalse(batchResponse.gets().get(1).failed()); + assertNull(batchResponse.gets().get(1).get()); + assertTrue(batchResponse.deletes().get(0).failed()); + assertTrue(batchResponse.deletes().get(1).failed()); assertTrue(storage.delete(BUCKET, blobName)); } From dc71a5605c58d5d25c1c0414848a7fc17035adb4 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 13 Nov 2015 10:28:16 -0800 Subject: [PATCH 42/94] Simplify package-info examples --- .../com/google/gcloud/datastore/package-info.java | 11 ++++++++--- .../java/com/google/gcloud/storage/package-info.java | 10 +++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java index 2135267d9ac4..405157ad55bd 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java @@ -17,10 +17,9 @@ /** * A client to the Google Cloud Datastore. * - *

A simple usage example: + *

Here's a simple usage example for using gcloud-java from App/Compute Engine: * 

 {@code
- * DatastoreOptions options = DatastoreOptions.builder().projectId(PROJECT_ID).build();
- * Datastore datastore = options.service();
+ * Datastore datastore = DatastoreOptions.defaultInstance().service();
  * KeyFactory keyFactory = datastore.newKeyFactory().kind(kind);
  * Key key = keyFactory.newKey(keyName);
  * Entity entity = datastore.get(key);
@@ -47,6 +46,12 @@
  * }
  * }
* + *

When using gcloud-java from outside of App/Compute Engine, you have to specify a + * project ID and + * provide + * authentication. + * * @see Google Cloud Datastore */ package com.google.gcloud.datastore; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java index 2a09631be40a..71a4e26da4ef 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java @@ -17,10 +17,9 @@ /** * A client to Google Cloud Storage. * - *

A simple usage example: + *

Here's a simple usage example for when running on App/Compute Engine: * 

{@code
- * StorageOptions options = StorageOptions.builder().projectId("project").build();
- * Storage storage = options.service();
+ * Storage storage = StorageOptions.defaultInstance().service();
  * BlobId blobId = BlobId.of("bucket", "blob_name");
  * Blob blob = Blob.load(storage, blobId);
  * if (blob == null) {
@@ -35,6 +34,11 @@
  *   channel.close();
  * }}
* + * When using gcloud-java from outside of App/Compute Engine, you have to specify a + * project ID and + * provide + * authentication. * @see Google Cloud Storage */ package com.google.gcloud.storage; From 22f52aee1eb371759e3b990e5bd70c5c12443621 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 13 Nov 2015 11:12:39 -0800 Subject: [PATCH 43/94] Document that we should wait until artifacts are pushed to finish release process --- RELEASING.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 419f723fe328..e9b2d3c2a0f8 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -10,13 +10,15 @@ This script takes an optional argument denoting the new version. By default, if 2. Create a PR to update the pom.xml version. The PR should look something like [#225](https://github.com/GoogleCloudPlatform/gcloud-java/pull/225). After this PR is merged into GoogleCloudPlatform/gcloud-java, Travis CI will push a new website to GoogleCloudPlatform/gh-pages, push a new artifact to the Maven Central Repository, and update versions in the README files. -3. Create a release on Github manually. +3. Before moving on, verify that the artifacts have successfully been pushed to the Maven Central Repository. To do this, we need to check the Travis CI logs since the artifacts take a couple hours to appear on the Maven Central Repository's website. Open Travis CI, click the ["Build History" tab](https://travis-ci.org/GoogleCloudPlatform/gcloud-java/builds), and open the second build's logs for Step 2's PR. Be sure that you are not opening the "Pull Request" build logs; you should not be under the "Pull Requests" tab. When the build finishes, scroll to the end of the log and verify that the artifacts were successfully staged and deployed. If the deployment didn't succeed because of a flaky test, the artifact may still have been pushed by the next Travis build, "Updating READMEs." If both builds failed to deploy the artifacts, rerun the build. + +4. Create a release on Github manually. Go to the [releases page](https://github.com/GoogleCloudPlatform/gcloud-java/releases) and click "Draft a new release." Use `vX.Y.Z` as the "Tag Version" and `X.Y.Z` as the "Release Title", where `X.Y.Z` is the release version as listed in the `pom.xml` files. -4. Run `utilities/update_pom_version.sh` again (to include "-SNAPSHOT" in the project version). +5. Run `utilities/update_pom_version.sh` again (to include "-SNAPSHOT" in the project version). As mentioned before, there is an optional version argument. By default, the script will update the version from "X.Y.Z" to "X.Y.Z+1-SNAPSHOT". Suppose a different version is desired, for example X+1.0.0-SNAPSHOT. Then the appropriate command to run would be `utilities/update_pom_version.sh X+1.0.0-SNAPSHOT`. -5. Create and merge in another PR to reflect the updated project version. For an example of what this PR should look like, see [#227](https://github.com/GoogleCloudPlatform/gcloud-java/pull/227). +6. Create and merge in another PR to reflect the updated project version. For an example of what this PR should look like, see [#227](https://github.com/GoogleCloudPlatform/gcloud-java/pull/227). ### To push a snapshot version From f3c82742ebd7c7989aafd06c3158831c6b8ebc80 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 13 Nov 2015 11:33:43 -0800 Subject: [PATCH 44/94] Fix code to get app engine ID --- .../src/main/java/com/google/gcloud/ServiceOptions.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java index 0793470ade83..c76b62780a0e 100644 --- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java @@ -450,9 +450,11 @@ protected static String getAppEngineProjectId() { try { Class factoryClass = Class.forName("com.google.appengine.api.appidentity.AppIdentityServiceFactory"); + Class serviceClass = + Class.forName("com.google.appengine.api.appidentity.AppIdentityService"); Method method = factoryClass.getMethod("getAppIdentityService"); Object appIdentityService = method.invoke(null); - method = appIdentityService.getClass().getMethod("getServiceAccountName"); + method = serviceClass.getMethod("getServiceAccountName"); String serviceAccountName = (String) method.invoke(appIdentityService); int indexOfAtSign = serviceAccountName.indexOf('@'); return serviceAccountName.substring(0, indexOfAtSign); From 1bcdf4b7ca7798e3d702a28d26d98ac60564b78d Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 13 Nov 2015 13:30:17 -0800 Subject: [PATCH 45/94] Make package info wording clearer --- .../main/java/com/google/gcloud/datastore/package-info.java | 3 +-- .../src/main/java/com/google/gcloud/storage/package-info.java | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java index 405157ad55bd..1404b2817802 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java @@ -50,8 +50,7 @@ * href="https://github.com/GoogleCloudPlatform/gcloud-java#specifying-a-project-id">specify a * project ID and * provide - * authentication. - * + * credentials. * @see Google Cloud Datastore */ package com.google.gcloud.datastore; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java index 71a4e26da4ef..137afd38b6ae 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java @@ -17,7 +17,7 @@ /** * A client to Google Cloud Storage. * - *

Here's a simple usage example for when running on App/Compute Engine: + *

Here's a simple usage example for using gcloud-java from App/Compute Engine: * 

{@code
  * Storage storage = StorageOptions.defaultInstance().service();
  * BlobId blobId = BlobId.of("bucket", "blob_name");
@@ -38,7 +38,7 @@
  * href="https://github.com/GoogleCloudPlatform/gcloud-java#specifying-a-project-id">specify a
  * project ID and
  * provide
- * authentication.
+ * credentials.
  * @see Google Cloud Storage
  */
 package com.google.gcloud.storage;

From cf777fcaa578ebeace11dff3b75e774c9874c9a9 Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Fri, 13 Nov 2015 18:50:28 +0100
Subject: [PATCH 46/94] Fix setting ContentRange to inclusive positions

---
 .../src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index b1e188f1d1fb..a033ed635d2d 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -60,6 +60,7 @@
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
+import com.google.common.primitives.Ints;
 import com.google.gcloud.storage.StorageException;
 import com.google.gcloud.storage.StorageOptions;
 
@@ -415,7 +416,7 @@ public byte[] read(StorageObject from, Map options, long position, in
           .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
           .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
       MediaHttpDownloader downloader = req.getMediaHttpDownloader();
-      downloader.setContentRange(position, (int) position + bytes);
+      downloader.setContentRange(position, Ints.checkedCast(position + bytes - 1));
       downloader.setDirectDownloadEnabled(true);
       ByteArrayOutputStream output = new ByteArrayOutputStream();
       req.executeMediaAndDownloadTo(output);

From 77dab2c0e5454a8ee33e1ef0f866dab32cc66a84 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Fri, 13 Nov 2015 16:42:44 -0800
Subject: [PATCH 47/94] Remove extra central repository deploy

---
 RELEASING.md                     | 2 +-
 utilities/update_docs_version.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/RELEASING.md b/RELEASING.md
index e9b2d3c2a0f8..7de5d599a395 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -10,7 +10,7 @@ This script takes an optional argument denoting the new version.  By default, if
 2. Create a PR to update the pom.xml version.
 The PR should look something like [#225](https://github.com/GoogleCloudPlatform/gcloud-java/pull/225).  After this PR is merged into GoogleCloudPlatform/gcloud-java, Travis CI will push a new website to GoogleCloudPlatform/gh-pages, push a new artifact to the Maven Central Repository, and update versions in the README files.
 
-3. Before moving on, verify that the artifacts have successfully been pushed to the Maven Central Repository.  To do this, we need to check the Travis CI logs since the artifacts take a couple hours to appear on the Maven Central Repository's website.  Open Travis CI, click the ["Build History" tab](https://travis-ci.org/GoogleCloudPlatform/gcloud-java/builds), and open the second build's logs for Step 2's PR.  Be sure that you are not opening the "Pull Request" build logs; you should not be under the "Pull Requests" tab.  When the build finishes, scroll to the end of the log and verify that the artifacts were successfully staged and deployed.  If the deployment didn't succeed because of a flaky test, the artifact may still have been pushed by the next Travis build, "Updating READMEs."  If both builds failed to deploy the artifacts, rerun the build.
+3. Before moving on, verify that the artifacts have successfully been pushed to the Maven Central Repository.  To do this, we need to check the Travis CI logs since the artifacts take a couple hours to appear on the Maven Central Repository's website.  Open Travis CI, click the ["Build History" tab](https://travis-ci.org/GoogleCloudPlatform/gcloud-java/builds), and open the second build's logs for Step 2's PR.  Be sure that you are not opening the "Pull Request" build logs; you should not be under the "Pull Requests" tab.  When the build finishes, scroll to the end of the log and verify that the artifacts were successfully staged and deployed.  If the deployment didn't succeed because of a flaky test, rerun the build.
 
 4. Create a release on Github manually.
 Go to the [releases page](https://github.com/GoogleCloudPlatform/gcloud-java/releases) and click "Draft a new release."  Use `vX.Y.Z` as the "Tag Version" and `X.Y.Z` as the "Release Title", where `X.Y.Z` is the release version as listed in the `pom.xml` files.
diff --git a/utilities/update_docs_version.sh b/utilities/update_docs_version.sh
index 4b1641a0bd81..4fc0aa772963 100755
--- a/utilities/update_docs_version.sh
+++ b/utilities/update_docs_version.sh
@@ -21,6 +21,6 @@ if [ "${RELEASED_VERSION##*-}" != "SNAPSHOT" ]; then
     git add README.md */README.md
     git config --global user.name "travis-ci"
     git config --global user.email "travis@travis-ci.org"
-    git commit -m "Updating version in README files."
+    git commit -m "Updating version in README files. [ci skip]"
     git push --quiet "https://${CI_DEPLOY_USERNAME}:${CI_DEPLOY_PASSWORD}@github.com/GoogleCloudPlatform/gcloud-java.git" HEAD:master > /dev/null 2>&1
 fi

From 3f6d791ce734a5b2f16064f2f4c07f1ede22fa73 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Fri, 13 Nov 2015 17:10:36 -0800
Subject: [PATCH 48/94] Add additional deploy check comment

---
 RELEASING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/RELEASING.md b/RELEASING.md
index 7de5d599a395..59b6559b9f95 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -10,7 +10,7 @@ This script takes an optional argument denoting the new version.  By default, if
 2. Create a PR to update the pom.xml version.
 The PR should look something like [#225](https://github.com/GoogleCloudPlatform/gcloud-java/pull/225).  After this PR is merged into GoogleCloudPlatform/gcloud-java, Travis CI will push a new website to GoogleCloudPlatform/gh-pages, push a new artifact to the Maven Central Repository, and update versions in the README files.
 
-3. Before moving on, verify that the artifacts have successfully been pushed to the Maven Central Repository.  To do this, we need to check the Travis CI logs since the artifacts take a couple hours to appear on the Maven Central Repository's website.  Open Travis CI, click the ["Build History" tab](https://travis-ci.org/GoogleCloudPlatform/gcloud-java/builds), and open the second build's logs for Step 2's PR.  Be sure that you are not opening the "Pull Request" build logs; you should not be under the "Pull Requests" tab.  When the build finishes, scroll to the end of the log and verify that the artifacts were successfully staged and deployed.  If the deployment didn't succeed because of a flaky test, rerun the build.
+3. Before moving on, verify that the artifacts have successfully been pushed to the Maven Central Repository.  Open Travis CI, click the ["Build History" tab](https://travis-ci.org/GoogleCloudPlatform/gcloud-java/builds), and open the second build's logs for Step 2's PR.  Be sure that you are not opening the "Pull Request" build logs.  When the build finishes, scroll to the end of the log and verify that the artifacts were successfully staged and deployed.  You can also search for `gcloud-java` on the [Sonatype website](https://oss.sonatype.org/#nexus-search;quick~gcloud-java) and check the latest version number.  If the deployment didn't succeed because of a flaky test, rerun the build.
 
 4. Create a release on Github manually.
 Go to the [releases page](https://github.com/GoogleCloudPlatform/gcloud-java/releases) and click "Draft a new release."  Use `vX.Y.Z` as the "Tag Version" and `X.Y.Z` as the "Release Title", where `X.Y.Z` is the release version as listed in the `pom.xml` files.

From 76cb620fc814083926e03bd981f7aefbf316f4ff Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Tue, 17 Nov 2015 15:30:39 +0100
Subject: [PATCH 49/94] Better document delete methods

---
 .../main/java/com/google/gcloud/storage/Blob.java    |  6 +++---
 .../main/java/com/google/gcloud/storage/Bucket.java  |  2 +-
 .../main/java/com/google/gcloud/storage/Storage.java | 12 ++++++------
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java
index a8e315be0e45..503cad361e29 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java
@@ -281,7 +281,7 @@ public CopyWriter copyTo(BlobId targetBlob, BlobSourceOption... options) {
    * Deletes this blob.
    *
    * @param options blob delete options
-   * @return true if blob was deleted
+   * @return {@code true} if blob was deleted, {@code false} if it was not found
    * @throws StorageException upon failure
    */
   public boolean delete(BlobSourceOption... options) {
@@ -422,8 +422,8 @@ public Blob apply(BlobInfo f) {
    * @param storage the storage service used to issue the request
    * @param blobs the blobs to delete
    * @return an immutable list of booleans. If a blob has been deleted the corresponding item in the
-   *     list is {@code true}. If deletion failed or access to the resource was denied the item is
-   *     {@code false}.
+   *     list is {@code true}. If a blob was not found, deletion failed or access to the resource
+   *     was denied the corresponding item is {@code false}.
    * @throws StorageException upon failure
    */
   public static List delete(Storage storage, BlobId... blobs) {
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java
index 21aafd92b5d4..54e60cc006dc 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -271,7 +271,7 @@ public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) {
    * Deletes this bucket.
    *
    * @param options bucket delete options
-   * @return true if bucket was deleted
+   * @return {@code true} if bucket was deleted, {@code false} if it was not found
    * @throws StorageException upon failure
    */
   public boolean delete(BucketSourceOption... options) {
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
index b61cfb269c1a..1fe096524ab7 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
@@ -1316,7 +1316,7 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx
   /**
    * Delete the requested bucket.
    *
-   * @return true if bucket was deleted
+   * @return {@code true} if bucket was deleted, {@code false} if it was not found
    * @throws StorageException upon failure
    */
   boolean delete(String bucket, BucketSourceOption... options);
@@ -1324,7 +1324,7 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx
   /**
    * Delete the requested blob.
    *
-   * @return true if blob was deleted
+   * @return {@code true} if blob was deleted, {@code false} if it was not found
    * @throws StorageException upon failure
    */
   boolean delete(String bucket, String blob, BlobSourceOption... options);
@@ -1332,7 +1332,7 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx
   /**
    * Delete the requested blob.
    *
-   * @return true if blob was deleted
+   * @return {@code true} if blob was deleted, {@code false} if it was not found
    * @throws StorageException upon failure
    */
   boolean delete(BlobId blob, BlobSourceOption... options);
@@ -1340,7 +1340,7 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx
   /**
    * Delete the requested blob.
    *
-   * @return true if blob was deleted
+   * @return {@code true} if blob was deleted, {@code false} if it was not found
    * @throws StorageException upon failure
    */
   boolean delete(BlobId blob);
@@ -1478,8 +1478,8 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx
    *
    * @param blobIds blobs to delete
    * @return an immutable list of booleans. If a blob has been deleted the corresponding item in the
-   *     list is {@code true}. If deletion failed or access to the resource was denied the item is
-   *     {@code false}.
+   *     list is {@code true}. If a blob was not found, deletion failed or access to the resource
+   *     was denied the corresponding item is {@code false}.
    * @throws StorageException upon failure
    */
   List delete(BlobId... blobIds);

From e22e8a3e6d6052c831f9e9f4237f428a7bb7558a Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Tue, 17 Nov 2015 16:55:42 +0100
Subject: [PATCH 50/94] Make batch delete return false if not found

---
 .../main/java/com/google/gcloud/spi/DefaultStorageRpc.java  | 6 +++++-
 .../test/java/com/google/gcloud/storage/ITStorageTest.java  | 3 ++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 55bca2319c2d..1021a58f64fe 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -368,7 +368,11 @@ public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
 
           @Override
           public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
-            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
+            if (e.getCode() == 404) {
+              deletes.put(tuple.x(), Tuple.of(Boolean.FALSE, null));
+            } else {
+              deletes.put(tuple.x(), Tuple.of(null, translate(e)));
+            }
           }
         });
       }
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
index d22bf06c8b16..0d4e876271e3 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
@@ -644,7 +644,8 @@ public void testBatchRequestFail() {
     assertFalse(batchResponse.gets().get(1).failed());
     assertNull(batchResponse.gets().get(1).get());
     assertTrue(batchResponse.deletes().get(0).failed());
-    assertTrue(batchResponse.deletes().get(1).failed());
+    assertFalse(batchResponse.deletes().get(1).failed());
+    assertFalse(batchResponse.deletes().get(1).get());
     assertTrue(storage.delete(BUCKET, blobName));
   }
 

From c3b6616c98982a201c1dcd6a5b1e7d3210d009d6 Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Mon, 9 Nov 2015 17:04:23 +0100
Subject: [PATCH 51/94] Use GOOGLE_APPLICATION_CREDENTIALS and GCLOUD_PROJECT
 vars in RemoteGcsHelper

---
 .../storage/testing/RemoteGcsHelper.java      | 57 +++++++------------
 utilities/integration_test_env.sh             |  4 +-
 2 files changed, 21 insertions(+), 40 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/RemoteGcsHelper.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/RemoteGcsHelper.java
index b15768cffa98..f5cdae83f999 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/RemoteGcsHelper.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/RemoteGcsHelper.java
@@ -45,8 +45,6 @@ public class RemoteGcsHelper {
 
   private static final Logger log = Logger.getLogger(RemoteGcsHelper.class.getName());
   private static final String BUCKET_NAME_PREFIX = "gcloud-test-bucket-temp-";
-  private static final String PROJECT_ID_ENV_VAR = "GCLOUD_TESTS_PROJECT_ID";
-  private static final String PRIVATE_KEY_ENV_VAR = "GCLOUD_TESTS_KEY";
   private final StorageOptions options;
 
   private RemoteGcsHelper(StorageOptions options) {
@@ -107,13 +105,7 @@ public static RemoteGcsHelper create(String projectId, InputStream keyStream)
       StorageOptions storageOptions = StorageOptions.builder()
           .authCredentials(AuthCredentials.createForJson(keyStream))
           .projectId(projectId)
-          .retryParams(RetryParams.builder()
-              .retryMaxAttempts(10)
-              .retryMinAttempts(6)
-              .maxRetryDelayMillis(30000)
-              .totalRetryPeriodMillis(120000)
-              .initialRetryDelayMillis(250)
-              .build())
+          .retryParams(retryParams())
           .connectTimeout(60000)
           .readTimeout(60000)
           .build();
@@ -145,41 +137,30 @@ public static RemoteGcsHelper create(String projectId, String keyPath)
         log.log(Level.WARNING, ex.getMessage());
       }
       throw GcsHelperException.translate(ex);
-    } catch (IOException ex) {
-      if (log.isLoggable(Level.WARNING)) {
-        log.log(Level.WARNING, ex.getMessage());
-      }
-      throw GcsHelperException.translate(ex);
     }
   }
 
   /**
-   * Creates a {@code RemoteGcsHelper} object. Project id and path to JSON key are read from two
-   * environment variables: {@code GCLOUD_TESTS_PROJECT_ID} and {@code GCLOUD_TESTS_KEY}.
-   *
-   * @return A {@code RemoteGcsHelper} object for the provided options.
-   * @throws com.google.gcloud.storage.testing.RemoteGcsHelper.GcsHelperException if environment
-   *     variables {@code GCLOUD_TESTS_PROJECT_ID} and {@code GCLOUD_TESTS_KEY} are not set or if
-   *     the file pointed by {@code GCLOUD_TESTS_KEY} does not exist
+   * Creates a {@code RemoteGcsHelper} object using default project id and authentication
+   * credentials.
    */
   public static RemoteGcsHelper create() throws GcsHelperException {
-    String projectId = System.getenv(PROJECT_ID_ENV_VAR);
-    String keyPath = System.getenv(PRIVATE_KEY_ENV_VAR);
-    if (projectId == null) {
-      String message = "Environment variable " + PROJECT_ID_ENV_VAR + " not set";
-      if (log.isLoggable(Level.WARNING)) {
-        log.log(Level.WARNING, message);
-      }
-      throw new GcsHelperException(message);
-    }
-    if (keyPath == null) {
-      String message = "Environment variable " + PRIVATE_KEY_ENV_VAR + " not set";
-      if (log.isLoggable(Level.WARNING)) {
-        log.log(Level.WARNING, message);
-      }
-      throw new GcsHelperException(message);
-    }
-    return create(projectId, keyPath);
+    StorageOptions storageOptions = StorageOptions.builder()
+        .retryParams(retryParams())
+        .connectTimeout(60000)
+        .readTimeout(60000)
+        .build();
+    return new RemoteGcsHelper(storageOptions);
+  }
+
+  private static RetryParams retryParams() {
+    return RetryParams.builder()
+        .retryMaxAttempts(10)
+        .retryMinAttempts(6)
+        .maxRetryDelayMillis(30000)
+        .totalRetryPeriodMillis(120000)
+        .initialRetryDelayMillis(250)
+        .build();
   }
 
   private static class DeleteBucketTask implements Callable {
diff --git a/utilities/integration_test_env.sh b/utilities/integration_test_env.sh
index f7aca1a8a623..a1bebe4dcb69 100755
--- a/utilities/integration_test_env.sh
+++ b/utilities/integration_test_env.sh
@@ -1,3 +1,3 @@
 # Export test env variables
-export GCLOUD_TESTS_PROJECT_ID="gcloud-devel"
-export GCLOUD_TESTS_KEY=$TRAVIS_BUILD_DIR/signing-tools/gcloud-devel-travis.json
+export GCLOUD_PROJECT="gcloud-devel"
+export GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/signing-tools/gcloud-devel-travis.json

From 82ebc7128133d7727eec8e85df927ec9599205f8 Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Tue, 17 Nov 2015 17:15:27 +0100
Subject: [PATCH 52/94] Add support for signUrl when default credentials are
 used - Add a method to convert ApplicationDefaultCredentials to
 ServiceAccountAuthCredentials - Add type check and conversion to
 Storage.signUrl

---
 .../java/com/google/gcloud/AuthCredentials.java    | 12 +++++++++++-
 .../com/google/gcloud/storage/StorageImpl.java     | 14 +++++++++++---
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
index 800fcf340689..afd785981ab9 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -28,6 +28,7 @@
 import com.google.api.client.json.jackson.JacksonFactory;
 import com.google.auth.http.HttpCredentialsAdapter;
 import com.google.auth.oauth2.GoogleCredentials;
+import com.google.auth.oauth2.ServiceAccountCredentials;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -212,7 +213,7 @@ public RestorableState capture() {
     }
   }
 
-  private static class ApplicationDefaultAuthCredentials extends AuthCredentials {
+  public static class ApplicationDefaultAuthCredentials extends AuthCredentials {
 
     private GoogleCredentials googleCredentials;
 
@@ -255,6 +256,15 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
       return new HttpCredentialsAdapter(googleCredentials.createScoped(scopes));
     }
 
+    public ServiceAccountAuthCredentials toServiceAccountCredentials() {
+      if (googleCredentials instanceof ServiceAccountCredentials) {
+        ServiceAccountCredentials credentials = (ServiceAccountCredentials) googleCredentials;
+        return new ServiceAccountAuthCredentials(credentials.getClientEmail(),
+            credentials.getPrivateKey());
+      }
+      return null;
+    }
+
     @Override
     public RestorableState capture() {
       return STATE;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index fa059254eddb..8ae3948a7576 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -43,6 +43,8 @@
 import com.google.common.hash.Hashing;
 import com.google.common.io.BaseEncoding;
 import com.google.common.primitives.Ints;
+import com.google.gcloud.AuthCredentials;
+import com.google.gcloud.AuthCredentials.ApplicationDefaultAuthCredentials;
 import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
 import com.google.gcloud.PageImpl;
 import com.google.gcloud.BaseService;
@@ -584,9 +586,15 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio
     ServiceAccountAuthCredentials cred =
         (ServiceAccountAuthCredentials) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED);
     if (cred == null) {
-      checkArgument(options().authCredentials() instanceof ServiceAccountAuthCredentials,
-          "Signing key was not provided and could not be derived");
-      cred = (ServiceAccountAuthCredentials) this.options().authCredentials();
+      AuthCredentials serviceCred = this.options().authCredentials();
+      if (serviceCred instanceof ServiceAccountAuthCredentials) {
+        cred = (ServiceAccountAuthCredentials) serviceCred;
+      } else {
+        if (serviceCred instanceof ApplicationDefaultAuthCredentials) {
+          cred = ((ApplicationDefaultAuthCredentials) serviceCred).toServiceAccountCredentials();
+        }
+      }
+      checkArgument(cred != null, "Signing key was not provided and could not be derived");
     }
     // construct signature - see https://cloud.google.com/storage/docs/access-control#Signed-URLs
     StringBuilder stBuilder = new StringBuilder();

From 93da8a9cde7af69c0ac21694266c5d238cbd6d70 Mon Sep 17 00:00:00 2001
From: Arie Ozarov 
Date: Tue, 17 Nov 2015 19:39:46 -0800
Subject: [PATCH 53/94] fix readmes for storage

---
 README.md                     | 3 +--
 gcloud-java-storage/README.md | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index bba06a7e60d4..03bf2e69f915 100644
--- a/README.md
+++ b/README.md
@@ -164,8 +164,7 @@ import com.google.gcloud.storage.StorageOptions;
 import java.nio.ByteBuffer;
 import java.nio.channels.WritableByteChannel;
 
-StorageOptions options = StorageOptions.builder().projectId("project").build();
-Storage storage = options.service();
+Storage storage = StorageOptions.defaultInstance().service();
 BlobId blobId = BlobId.of("bucket", "blob_name");
 Blob blob = Blob.load(storage, blobId);
 if (blob == null) {
diff --git a/gcloud-java-storage/README.md b/gcloud-java-storage/README.md
index 8b19484dd14d..0302439a4314 100644
--- a/gcloud-java-storage/README.md
+++ b/gcloud-java-storage/README.md
@@ -68,7 +68,7 @@ import com.google.gcloud.storage.StorageOptions;
 import java.nio.ByteBuffer;
 import java.nio.channels.WritableByteChannel;
 
-Storage storage = StorageOptions.getDefaultInstance().service();
+Storage storage = StorageOptions.defaultInstance().service();
 Blob blob = new Blob(storage, "bucket", "blob_name");
 if (!blob.exists()) {
   storage2.create(blob.info(), "Hello, Cloud Storage!".getBytes(UTF_8));

From 87e4bac0dff3ef1f36f67e6a119072edd0d3509c Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Wed, 18 Nov 2015 11:36:56 +0100
Subject: [PATCH 54/94] Fix BlobWriteChannel and BlobReadChannel javadoc

---
 .../main/java/com/google/gcloud/storage/BlobReadChannel.java | 5 ++---
 .../java/com/google/gcloud/storage/BlobWriteChannel.java     | 4 ++--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
index 205dc4b97309..54d39649cb70 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
@@ -26,9 +26,8 @@
 /**
  * A channel for reading data from a Google Cloud Storage object.
  *
- * Implementations of this class may buffer data internally to reduce remote calls.
- *
- * This class is @{link Serializable}, which allows incremental reads.
+ * Implementations of this class may buffer data internally to reduce remote calls. This interface
+ * implements {@link Restorable} to allow saving the reader's state to continue reading afterwards.
  */
 public interface BlobReadChannel extends ReadableByteChannel, Closeable,
     Restorable {
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
index a6208e5020ae..fe9164532120 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
@@ -26,8 +26,8 @@
  * A channel for writing data to a Google Cloud Storage object.
  *
  * Implementations of this class may further buffer data internally to reduce remote calls. Written
- * data will only be visible after calling {@link #close()}. This class is serializable, to allow
- * incremental writes.
+ * data will only be visible after calling {@link #close()}. This interface implements
+ * {@link Restorable} to allow saving the writer's state to continue writing afterwards.
  */
 public interface BlobWriteChannel extends WritableByteChannel, Closeable,
     Restorable {

From 65373146e8cd827011631eb111ad4a5b52f5c2a5 Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Wed, 18 Nov 2015 11:39:08 +0100
Subject: [PATCH 55/94] BlobReadChannel fails if blob is updated while being
 read - Change StorageRpc.read to return a pair (etag, bytes) - Keep track the
 last read etag in BlobReadChannelImpl to detect changes - Throw
 StorageException if new etag does not match the last one - Add unit and
 integration tests - Document this behavior in Storage.reader javadoc

---
 .../google/gcloud/spi/DefaultStorageRpc.java  | 15 +++---
 .../com/google/gcloud/spi/StorageRpc.java     |  2 +-
 .../gcloud/storage/BlobReadChannelImpl.java   | 27 +++++++++--
 .../com/google/gcloud/storage/Storage.java    | 11 ++++-
 .../storage/BlobReadChannelImplTest.java      | 48 ++++++++++++++-----
 .../google/gcloud/storage/ITStorageTest.java  | 34 +++++++++++++
 .../gcloud/storage/StorageImplTest.java       |  4 +-
 7 files changed, 114 insertions(+), 27 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 55bca2319c2d..7a636616d810 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -410,8 +410,8 @@ public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
   }
 
   @Override
-  public byte[] read(StorageObject from, Map options, long position, int bytes)
-      throws StorageException {
+  public Tuple read(StorageObject from, Map options, long position,
+      int bytes) throws StorageException {
     try {
       Get req = storage.objects()
           .get(from.getBucket(), from.getName())
@@ -420,12 +420,13 @@ public byte[] read(StorageObject from, Map options, long position, in
           .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
           .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
           .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
-      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
-      downloader.setContentRange(position, Ints.checkedCast(position + bytes - 1));
-      downloader.setDirectDownloadEnabled(true);
+      StringBuilder range = new StringBuilder();
+      range.append("bytes=").append(position).append("-").append(position + bytes - 1);
+      req.getRequestHeaders().setRange(range.toString());
       ByteArrayOutputStream output = new ByteArrayOutputStream();
-      req.executeMediaAndDownloadTo(output);
-      return output.toByteArray();
+      req.executeMedia().download(output);
+      String etag = req.getLastResponseHeaders().getETag();
+      return Tuple.of(etag, output.toByteArray());
     } catch (IOException ex) {
       throw translate(ex);
     }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
index e4b1be785951..2e85bd72bf05 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -249,7 +249,7 @@ StorageObject compose(Iterable sources, StorageObject target,
   byte[] load(StorageObject storageObject, Map options)
       throws StorageException;
 
-  byte[] read(StorageObject from, Map options, long position, int bytes)
+  Tuple read(StorageObject from, Map options, long position, int bytes)
       throws StorageException;
 
   String open(StorageObject object, Map options) throws StorageException;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
index 09047a642218..8fe6eae66d8f 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
@@ -23,6 +23,7 @@
 import com.google.gcloud.RestorableState;
 import com.google.gcloud.RetryHelper;
 import com.google.gcloud.spi.StorageRpc;
+import com.google.gcloud.spi.StorageRpc.Tuple;
 
 import java.io.IOException;
 import java.io.Serializable;
@@ -41,6 +42,7 @@ class BlobReadChannelImpl implements BlobReadChannel {
   private final StorageOptions serviceOptions;
   private final BlobId blob;
   private final Map requestOptions;
+  private String lastEtag;
   private int position;
   private boolean isOpen;
   private boolean endOfStream;
@@ -117,12 +119,19 @@ public int read(ByteBuffer byteBuffer) throws IOException {
       }
       final int toRead = Math.max(byteBuffer.remaining(), chunkSize);
       try {
-        buffer = runWithRetries(new Callable() {
+        Tuple result = runWithRetries(new Callable>() {
           @Override
-          public byte[] call() {
+          public Tuple call() {
             return storageRpc.read(storageObject, requestOptions, position, toRead);
           }
         }, serviceOptions.retryParams(), StorageImpl.EXCEPTION_HANDLER);
+        if (lastEtag != null && !Objects.equals(result.x(), lastEtag)) {
+          StringBuilder messageBuilder = new StringBuilder();
+          messageBuilder.append("Blob ").append(blob).append(" was updated while reading");
+          throw new StorageException(0, messageBuilder.toString(), false);
+        }
+        lastEtag = result.x();
+        buffer = result.y();
       } catch (RetryHelper.RetryHelperException e) {
         throw StorageException.translateAndThrow(e);
       }
@@ -152,6 +161,7 @@ static class StateImpl implements RestorableState, Serializable
     private final StorageOptions serviceOptions;
     private final BlobId blob;
     private final Map requestOptions;
+    private final String lastEtag;
     private final int position;
     private final boolean isOpen;
     private final boolean endOfStream;
@@ -161,6 +171,7 @@ static class StateImpl implements RestorableState, Serializable
       this.serviceOptions = builder.serviceOptions;
       this.blob = builder.blob;
       this.requestOptions = builder.requestOptions;
+      this.lastEtag = builder.lastEtag;
       this.position = builder.position;
       this.isOpen = builder.isOpen;
       this.endOfStream = builder.endOfStream;
@@ -171,6 +182,7 @@ static class Builder {
       private final StorageOptions serviceOptions;
       private final BlobId blob;
       private final Map requestOptions;
+      private String lastEtag;
       private int position;
       private boolean isOpen;
       private boolean endOfStream;
@@ -182,6 +194,11 @@ private Builder(StorageOptions options, BlobId blob, Map r
         this.requestOptions = reqOptions;
       }
 
+      Builder lastEtag(String lastEtag) {
+        this.lastEtag = lastEtag;
+        return this;
+      }
+
       Builder position(int position) {
         this.position = position;
         return this;
@@ -215,6 +232,7 @@ static Builder builder(
     @Override
     public BlobReadChannel restore() {
       BlobReadChannelImpl channel = new BlobReadChannelImpl(serviceOptions, blob, requestOptions);
+      channel.lastEtag = lastEtag;
       channel.position = position;
       channel.isOpen = isOpen;
       channel.endOfStream = endOfStream;
@@ -224,8 +242,8 @@ public BlobReadChannel restore() {
 
     @Override
     public int hashCode() {
-      return Objects.hash(serviceOptions, blob, requestOptions, position, isOpen, endOfStream,
-          chunkSize);
+      return Objects.hash(serviceOptions, blob, requestOptions, lastEtag, position, isOpen,
+          endOfStream, chunkSize);
     }
 
     @Override
@@ -240,6 +258,7 @@ public boolean equals(Object obj) {
       return Objects.equals(this.serviceOptions, other.serviceOptions)
           && Objects.equals(this.blob, other.blob)
           && Objects.equals(this.requestOptions, other.requestOptions)
+          && Objects.equals(this.lastEtag, other.lastEtag)
           && this.position == other.position
           && this.isOpen == other.isOpen
           && this.endOfStream == other.endOfStream
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
index b61cfb269c1a..301b71ec5fe9 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
@@ -33,6 +33,7 @@
 import java.io.InputStream;
 import java.io.Serializable;
 import java.net.URL;
+import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
@@ -1405,14 +1406,20 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx
   BatchResponse apply(BatchRequest batchRequest);
 
   /**
-   * Return a channel for reading the blob's content.
+   * Return a channel for reading the blob's content. The blob's latest generation is read. If the
+   * blob changes while reading (i.e. {@link BlobInfo#etag()} changes), subsequent calls to
+   * {@link BlobReadChannel#read(ByteBuffer)} may throw {@link StorageException}.
    *
    * @throws StorageException upon failure
    */
   BlobReadChannel reader(String bucket, String blob, BlobSourceOption... options);
 
   /**
-   * Return a channel for reading the blob's content.
+   * Return a channel for reading the blob's content. If {@code blob.generation()} is set
+   * data corresponding to that generation is read. If {@code blob.generation()} is {@code null}
+   * the blob's latest generation is read. If the blob changes while reading (i.e.
+   * {@link BlobInfo#etag()} changes), subsequent calls to {@link BlobReadChannel#read(ByteBuffer)}
+   * may throw {@link StorageException}.
    *
    * @throws StorageException upon failure
    */
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java
index e1f904bf72fe..fe0f7a02ae4e 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java
@@ -19,7 +19,6 @@
 import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.verify;
 import static org.junit.Assert.assertArrayEquals;
@@ -46,7 +45,7 @@ public class BlobReadChannelImplTest {
 
   private static final String BUCKET_NAME = "b";
   private static final String BLOB_NAME = "n";
-  private static final BlobId BLOB_ID = BlobId.of(BUCKET_NAME, BLOB_NAME);
+  private static final BlobId BLOB_ID = BlobId.of(BUCKET_NAME, BLOB_NAME, -1L);
   private static final Map EMPTY_RPC_OPTIONS = ImmutableMap.of();
   private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024;
   private static final int CUSTOM_CHUNK_SIZE = 2 * 1024 * 1024;
@@ -88,7 +87,7 @@ public void testReadBuffered() throws IOException {
     ByteBuffer firstReadBuffer = ByteBuffer.allocate(42);
     ByteBuffer secondReadBuffer = ByteBuffer.allocate(42);
     expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
-        .andReturn(result);
+        .andReturn(StorageRpc.Tuple.of("etag", result));
     replay(storageRpcMock);
     reader.read(firstReadBuffer);
     reader.read(secondReadBuffer);
@@ -107,10 +106,11 @@ public void testReadBig() throws IOException {
     byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE);
     ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
     ByteBuffer secondReadBuffer = ByteBuffer.allocate(42);
-    storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE);
-    expectLastCall().andReturn(firstResult);
-    storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, CUSTOM_CHUNK_SIZE);
-    expectLastCall().andReturn(secondResult);
+    expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
+        .andReturn(StorageRpc.Tuple.of("etag", firstResult));
+    expect(storageRpcMock.read(
+        BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, CUSTOM_CHUNK_SIZE))
+            .andReturn(StorageRpc.Tuple.of("etag", secondResult));
     replay(storageRpcMock);
     reader.read(firstReadBuffer);
     reader.read(secondReadBuffer);
@@ -125,7 +125,7 @@ public void testReadFinish() throws IOException {
     byte[] result = {};
     ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
     expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
-        .andReturn(result);
+        .andReturn(StorageRpc.Tuple.of("etag", result));
     replay(storageRpcMock);
     assertEquals(-1, reader.read(readBuffer));
   }
@@ -137,7 +137,7 @@ public void testSeek() throws IOException {
     byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE);
     ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
     expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE))
-        .andReturn(result);
+        .andReturn(StorageRpc.Tuple.of("etag", result));
     replay(storageRpcMock);
     reader.read(readBuffer);
     assertArrayEquals(result, readBuffer.array());
@@ -166,6 +166,32 @@ public void testReadClosed() {
     }
   }
 
+  @Test
+  public void testReadGenerationChanged() throws IOException {
+    BlobId blobId = BlobId.of(BUCKET_NAME, BLOB_NAME);
+    reader = new BlobReadChannelImpl(options, blobId, EMPTY_RPC_OPTIONS);
+    byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE);
+    byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE);
+    ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
+    ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
+    expect(storageRpcMock.read(blobId.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
+        .andReturn(StorageRpc.Tuple.of("etag1", firstResult));
+    expect(storageRpcMock.read(
+        blobId.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_SIZE))
+            .andReturn(StorageRpc.Tuple.of("etag2", firstResult));
+    replay(storageRpcMock);
+    reader.read(firstReadBuffer);
+    try {
+      reader.read(secondReadBuffer);
+      fail("Expected BlobReadChannel read to throw StorageException");
+    } catch (StorageException ex) {
+      StringBuilder messageBuilder = new StringBuilder();
+      messageBuilder.append("Blob ").append(blobId).append(" was updated while reading");
+      assertEquals(messageBuilder.toString(), ex.getMessage());
+      // expected
+    }
+  }
+
   @Test
   public void testSaveAndRestore() throws IOException, ClassNotFoundException {
     byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE);
@@ -173,9 +199,9 @@ public void testSaveAndRestore() throws IOException, ClassNotFoundException {
     ByteBuffer firstReadBuffer = ByteBuffer.allocate(42);
     ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
     expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
-        .andReturn(firstResult);
+        .andReturn(StorageRpc.Tuple.of("etag", firstResult));
     expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE))
-        .andReturn(secondResult);
+        .andReturn(StorageRpc.Tuple.of("etag", secondResult));
     replay(storageRpcMock);
     reader = new BlobReadChannelImpl(options, BLOB_ID, EMPTY_RPC_OPTIONS);
     reader.read(firstReadBuffer);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
index d22bf06c8b16..c5f4fc09bcba 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
@@ -49,6 +49,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -730,6 +731,39 @@ public void testReadChannelFail() throws IOException {
     assertTrue(storage.delete(BUCKET, blobName));
   }
 
+  @Test
+  public void testReadChannelFailUpdatedGeneration() throws IOException {
+    String blobName = "test-read-blob-fail-updated-generation";
+    BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
+    Random random = new Random();
+    int chunkSize = 1024;
+    int blobSize = 2 * chunkSize;
+    byte[] content = new byte[blobSize];
+    random.nextBytes(content);
+    BlobInfo remoteBlob = storage.create(blob, content);
+    assertNotNull(remoteBlob);
+    assertEquals(blobSize, (long) remoteBlob.size());
+    try (BlobReadChannel reader = storage.reader(blob.blobId())) {
+      reader.chunkSize(chunkSize);
+      ByteBuffer readBytes = ByteBuffer.allocate(chunkSize);
+      int numReadBytes = reader.read(readBytes);
+      assertEquals(chunkSize, numReadBytes);
+      assertArrayEquals(Arrays.copyOf(content, chunkSize), readBytes.array());
+      try (BlobWriteChannel writer = storage.writer(blob)) {
+        byte[] newContent = new byte[blobSize];
+        random.nextBytes(newContent);
+        int numWrittenBytes = writer.write(ByteBuffer.wrap(newContent));
+        assertEquals(blobSize, numWrittenBytes);
+      }
+      readBytes = ByteBuffer.allocate(chunkSize);
+      reader.read(readBytes);
+      fail("StorageException was expected");
+    } catch(StorageException ex) {
+      // expected
+    }
+    assertTrue(storage.delete(BUCKET, blobName));
+  }
+
   @Test
   public void testWriteChannelFail() throws IOException {
     String blobName = "test-write-channel-blob-fail";
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
index 359920314e82..32a466a9d551 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
@@ -1030,7 +1030,7 @@ public void testReaderWithOptions() throws IOException {
     byte[] result = new byte[DEFAULT_CHUNK_SIZE];
     EasyMock.expect(
         storageRpcMock.read(BLOB_INFO2.toPb(), BLOB_SOURCE_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
-        .andReturn(result);
+        .andReturn(StorageRpc.Tuple.of("etag", result));
     EasyMock.replay(storageRpcMock);
     storage = options.service();
     BlobReadChannel channel = storage.reader(BUCKET_NAME1, BLOB_NAME2, BLOB_SOURCE_GENERATION,
@@ -1045,7 +1045,7 @@ public void testReaderWithOptionsFromBlobId() throws IOException {
     byte[] result = new byte[DEFAULT_CHUNK_SIZE];
     EasyMock.expect(
         storageRpcMock.read(BLOB_INFO1.blobId().toPb(), BLOB_SOURCE_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
-        .andReturn(result);
+        .andReturn(StorageRpc.Tuple.of("etag", result));
     EasyMock.replay(storageRpcMock);
     storage = options.service();
     BlobReadChannel channel = storage.reader(BLOB_INFO1.blobId(),

From 1e7fe54e8cef571608ab4992c2769b5cad5c615b Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Wed, 18 Nov 2015 10:16:37 +0100
Subject: [PATCH 56/94] Move all checks for HTTP_NOT_FOUND to DefaultStorageRpc

---
 .../google/gcloud/spi/DefaultStorageRpc.java  | 31 ++++++++++----
 .../com/google/gcloud/spi/StorageRpc.java     | 10 +++++
 .../google/gcloud/storage/StorageImpl.java    | 40 +++++--------------
 3 files changed, 43 insertions(+), 38 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 1021a58f64fe..0bd80a9c11ce 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -30,6 +30,7 @@
 import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
 import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
 import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;
+import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
 
 import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
 import com.google.api.client.googleapis.json.GoogleJsonError;
@@ -94,7 +95,8 @@ public DefaultStorageRpc(StorageOptions options) {
 
   private static StorageException translate(IOException exception) {
     StorageException translated;
-    if (exception instanceof GoogleJsonResponseException) {
+    if (exception instanceof GoogleJsonResponseException
+        && ((GoogleJsonResponseException) exception).getDetails() != null) {
       translated = translate(((GoogleJsonResponseException) exception).getDetails());
     } else {
       translated = new StorageException(0, exception.getMessage(), false);
@@ -191,7 +193,11 @@ public Bucket get(Bucket bucket, Map options) {
           .setFields(FIELDS.getString(options))
           .execute();
     } catch (IOException ex) {
-      throw translate(ex);
+      StorageException serviceException = translate(ex);
+      if (serviceException.code() == HTTP_NOT_FOUND) {
+        return null;
+      }
+      throw serviceException;
     }
   }
 
@@ -200,7 +206,11 @@ public StorageObject get(StorageObject object, Map options) {
     try {
       return getRequest(object, options).execute();
     } catch (IOException ex) {
-      throw translate(ex);
+      StorageException serviceException = translate(ex);
+      if (serviceException.code() == HTTP_NOT_FOUND) {
+        return null;
+      }
+      throw serviceException;
     }
   }
 
@@ -265,7 +275,7 @@ public boolean delete(Bucket bucket, Map options) {
       return true;
     } catch (IOException ex) {
       StorageException serviceException = translate(ex);
-      if (serviceException.code() == 404) {
+      if (serviceException.code() == HTTP_NOT_FOUND) {
         return false;
       }
       throw serviceException;
@@ -279,7 +289,7 @@ public boolean delete(StorageObject blob, Map options) {
       return true;
     } catch (IOException ex) {
       StorageException serviceException = translate(ex);
-      if (serviceException.code() == 404) {
+      if (serviceException.code() == HTTP_NOT_FOUND) {
         return false;
       }
       throw serviceException;
@@ -368,7 +378,7 @@ public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
 
           @Override
           public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
-            if (e.getCode() == 404) {
+            if (e.getCode() == HTTP_NOT_FOUND) {
               deletes.put(tuple.x(), Tuple.of(Boolean.FALSE, null));
             } else {
               deletes.put(tuple.x(), Tuple.of(null, translate(e)));
@@ -401,8 +411,13 @@ public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders)
 
           @Override
           public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
-            gets.put(tuple.x(),
-                Tuple.of(null, translate(e)));
+            if (e.getCode() == HTTP_NOT_FOUND) {
+              gets.put(tuple.x(),
+                  Tuple.of(null, null));
+            } else {
+              gets.put(tuple.x(),
+                  Tuple.of(null, translate(e)));
+            }
           }
         });
       }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
index e4b1be785951..1f708c269ad0 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -227,8 +227,18 @@ StorageObject create(StorageObject object, InputStream content, Map o
   Tuple> list(String bucket, Map options)
       throws StorageException;
 
+  /**
+   * Returns the requested bucket or {@code null} if not found.
+   *
+   * @throws StorageException upon failure
+   */
   Bucket get(Bucket bucket, Map options) throws StorageException;
 
+  /**
+   * Returns the requested storage object or {@code null} if not found.
+   *
+   * @throws StorageException upon failure
+   */
   StorageObject get(StorageObject object, Map options)
       throws StorageException;
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index fa059254eddb..0f8d9124a4cf 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -28,7 +28,6 @@
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
-import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.services.storage.model.StorageObject;
@@ -175,14 +174,7 @@ public BucketInfo get(String bucket, BucketGetOption... options) {
           new Callable() {
             @Override
             public com.google.api.services.storage.model.Bucket call() {
-              try {
-                return storageRpc.get(bucketPb, optionsMap);
-              } catch (StorageException ex) {
-                if (ex.code() == HTTP_NOT_FOUND) {
-                  return null;
-                }
-                throw ex;
-              }
+              return storageRpc.get(bucketPb, optionsMap);
             }
           }, options().retryParams(), EXCEPTION_HANDLER);
       return answer == null ? null : BucketInfo.fromPb(answer);
@@ -204,14 +196,7 @@ public BlobInfo get(BlobId blob, BlobGetOption... options) {
       StorageObject storageObject = runWithRetries(new Callable() {
         @Override
         public StorageObject call() {
-          try {
-            return storageRpc.get(storedObject, optionsMap);
-          } catch (StorageException ex) {
-            if (ex.code() == HTTP_NOT_FOUND) {
-              return null;
-            }
-            throw ex;
-          }
+          return storageRpc.get(storedObject, optionsMap);
         }
       }, options().retryParams(), EXCEPTION_HANDLER);
       return storageObject == null ? null : BlobInfo.fromPb(storageObject);
@@ -523,28 +508,23 @@ public BatchResponse apply(BatchRequest batchRequest) {
     List> updates = transformBatchResult(
         toUpdate, response.updates, BlobInfo.FROM_PB_FUNCTION);
     List> gets = transformBatchResult(
-        toGet, response.gets, BlobInfo.FROM_PB_FUNCTION, HTTP_NOT_FOUND);
+        toGet, response.gets, BlobInfo.FROM_PB_FUNCTION);
     return new BatchResponse(deletes, updates, gets);
   }
 
   private  List> transformBatchResult(
       Iterable>> request,
-      Map> results, Function transform,
-      int... nullOnErrorCodes) {
-    Set nullOnErrorCodesSet = Sets.newHashSet(Ints.asList(nullOnErrorCodes));
+      Map> results, Function transform) {
     List> response = Lists.newArrayListWithCapacity(results.size());
     for (Tuple tuple : request) {
       Tuple result = results.get(tuple.x());
-      if (result.x() != null) {
-        response.add(BatchResponse.Result.of(transform.apply(result.x())));
+      I object = result.x();
+      StorageException exception = result.y();
+      if (exception != null) {
+        response.add(new BatchResponse.Result(exception));
       } else {
-        StorageException exception = result.y();
-        if (nullOnErrorCodesSet.contains(exception.code())) {
-          //noinspection unchecked
-          response.add(BatchResponse.Result.empty());
-        } else {
-          response.add(new BatchResponse.Result(exception));
-        }
+        response.add(object != null ?
+            BatchResponse.Result.of(transform.apply(object)) : BatchResponse.Result.empty());
       }
     }
     return response;

From a5607774aee109dea6de6202a8f9ffb6652bcb6f Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Wed, 18 Nov 2015 15:28:48 -0800
Subject: [PATCH 57/94] Use version auth library verison 0.3.1 to avoid NPE

---
 gcloud-java-core/pom.xml | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml
index e849594226db..ea10f5145800 100644
--- a/gcloud-java-core/pom.xml
+++ b/gcloud-java-core/pom.xml
@@ -20,12 +20,29 @@
     
       com.google.auth
       google-auth-library-credentials
-      0.1.0
+      0.3.1
     
     
       com.google.auth
       google-auth-library-oauth2-http
-      0.1.0
+      0.3.1
+      
+        
+          com.google.guava
+          guava-jdk5
+        
+      
+    
+    
+      com.google.auth
+      google-auth-library-appengine
+      0.3.1
+      
+        
+          com.google.guava
+          guava-jdk5
+        
+      
     
     
       com.google.http-client

From 02694ca9280c9b0c482afcdd10d98b8afe42d50f Mon Sep 17 00:00:00 2001
From: Arie Ozarov 
Date: Wed, 18 Nov 2015 18:05:05 -0800
Subject: [PATCH 58/94] Update javadoc for FullEntity

---
 .../src/main/java/com/google/gcloud/datastore/FullEntity.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java
index bb08fca12e3c..55c573b9a636 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java
@@ -19,7 +19,8 @@
 import com.google.api.services.datastore.DatastoreV1;
 
 /**
- * A full entity is a {@link BaseEntity} that with a complete set of properties.
+ * A full entity is a {@link BaseEntity} that holds all the properties associated with a
+ * Datastore entity (as opposed to {@link ProjectionEntity}).
  */
 public class FullEntity extends BaseEntity {
 

From bfe892428127615e2d75bcc51378c125e76ab637 Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Thu, 19 Nov 2015 07:41:53 +0100
Subject: [PATCH 59/94] Add message check to IT tests, better javadoc for
 storage.reader

---
 .../java/com/google/gcloud/storage/Storage.java     | 13 +++++++++++--
 .../gcloud/storage/BlobReadChannelImplTest.java     |  1 -
 .../com/google/gcloud/storage/ITStorageTest.java    |  4 +++-
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
index 301b71ec5fe9..68a1c3baa3cf 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
@@ -1408,7 +1408,11 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx
   /**
    * Return a channel for reading the blob's content. The blob's latest generation is read. If the
    * blob changes while reading (i.e. {@link BlobInfo#etag()} changes), subsequent calls to
-   * {@link BlobReadChannel#read(ByteBuffer)} may throw {@link StorageException}.
+   * {@code blobReadChannel.read(ByteBuffer)} may throw {@link StorageException}.
+   *
+   * 
The {@link BlobSourceOption#generationMatch(long)} option can be provided to ensure that
+   * {@code blobReadChannel.read(ByteBuffer)} calls will throw {@link StorageException} if blob`s
+   * generation differs from the expected one.
    *
    * @throws StorageException upon failure
    */
@@ -1418,9 +1422,14 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx
    * Return a channel for reading the blob's content. If {@code blob.generation()} is set
    * data corresponding to that generation is read. If {@code blob.generation()} is {@code null}
    * the blob's latest generation is read. If the blob changes while reading (i.e.
-   * {@link BlobInfo#etag()} changes), subsequent calls to {@link BlobReadChannel#read(ByteBuffer)}
+   * {@link BlobInfo#etag()} changes), subsequent calls to {@code blobReadChannel.read(ByteBuffer)}
    * may throw {@link StorageException}.
    *
+   * 
The {@link BlobSourceOption#generationMatch()} and
+   * {@link BlobSourceOption#generationMatch(long)} options can be used to ensure that
+   * {@code blobReadChannel.read(ByteBuffer)} calls will throw {@link StorageException} if the
+   * blob`s generation differs from the expected one.
+   *
    * @throws StorageException upon failure
    */
   BlobReadChannel reader(BlobId blob, BlobSourceOption... options);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java
index fe0f7a02ae4e..f99fe893d0d9 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java
@@ -188,7 +188,6 @@ public void testReadGenerationChanged() throws IOException {
       StringBuilder messageBuilder = new StringBuilder();
       messageBuilder.append("Blob ").append(blobId).append(" was updated while reading");
       assertEquals(messageBuilder.toString(), ex.getMessage());
-      // expected
     }
   }
 
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
index c5f4fc09bcba..e76ebf54eb2d 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
@@ -759,7 +759,9 @@ public void testReadChannelFailUpdatedGeneration() throws IOException {
       reader.read(readBytes);
       fail("StorageException was expected");
     } catch(StorageException ex) {
-      // expected
+      StringBuilder messageBuilder = new StringBuilder();
+      messageBuilder.append("Blob ").append(blob.blobId()).append(" was updated while reading");
+      assertEquals(messageBuilder.toString(), ex.getMessage());
     }
     assertTrue(storage.delete(BUCKET, blobName));
   }

From a07fbda2724b3faef46b43a251aea0ac9b42625a Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Thu, 19 Nov 2015 14:23:05 -0800
Subject: [PATCH 60/94] Add step-by-step datastore guide

---
 gcloud-java-datastore/README.md | 141 +++++++++++++++++++++++++++-----
 1 file changed, 122 insertions(+), 19 deletions(-)

diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md
index 2525413ecb86..66cea4ef2037 100644
--- a/gcloud-java-datastore/README.md
+++ b/gcloud-java-datastore/README.md
@@ -56,33 +56,136 @@ Cloud Datastore for your project.
 See the ``gcloud-java`` API [datastore documentation][datastore-api] to learn how to interact
 with the Cloud Datastore using this Client Library.
 
-Here is a code snippet showing a simple usage example from within Compute/App Engine.  Note that you must [supply credentials](https://github.com/GoogleCloudPlatform/gcloud-java#authentication) and a project ID if running this snippet elsewhere.
+Getting Started
+---------------
+#### Prerequisites
+For this tutorial, you will need a [Google Developers Console](https://console.developers.google.com/) project with the Datastore API enabled. [Follow these instructions](https://cloud.google.com/docs/authentication#preparation) to get your project set up. You will also need to set up the local development environment by [installing the Google Cloud SDK](https://cloud.google.com/sdk/) and running the following commands in command line: `gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`.
+
+#### Installation and setup
+You'll need to obtain the `gcloud-java-datastore` library.  See the [Quickstart](#quickstart) section to add `gcloud-java-datastore` as a dependency in your code.
+
+#### Creating an authorized service object
+To make authenticated requests to Google Cloud Datastore, you must create a service object with credentials. You can then make API calls by calling methods on the Datastore service object. The simplest way to authenticate is to use [Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials). These credentials are automatically inferred from your environment, so you only need the following code to create your service object:
 
 ```java
 import com.google.gcloud.datastore.Datastore;
 import com.google.gcloud.datastore.DatastoreOptions;
-import com.google.gcloud.datastore.DateTime;
+
+Datastore datastore = DatastoreOptions.defaultInstance().service();
+```
+
+For other authentication options, see the [Authentication](https://github.com/GoogleCloudPlatform/gcloud-java#authentication) page.
+
+#### Storing data
+Objects in Datastore are known as entities. Entities are grouped by "kind" and have keys for easy access. In this code snippet, we will create a new entity representing a person and store that data by the person's name.  First, add the following imports at the top of your file:
+
+```java
 import com.google.gcloud.datastore.Entity;
 import com.google.gcloud.datastore.Key;
 import com.google.gcloud.datastore.KeyFactory;
+```
 
-Datastore datastore = DatastoreOptions.defaultInstance().service();
-KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND);
-Key key = keyFactory.newKey(keyName);
-Entity entity = datastore.get(key);
-if (entity == null) {
-  entity = Entity.builder(key)
-      .set("name", "John Do")
-      .set("age", 30)
-      .set("access_time", DateTime.now())
-      .build();
-  datastore.put(entity);
-} else {
-  System.out.println("Updating access_time for " + entity.getString("name"));
-  entity = Entity.builder(entity)
-      .set("access_time", DateTime.now())
-      .build();
-  datastore.update(entity);
+Then add the following code to put an entity in Datastore.
+
+```java
+KeyFactory keyFactory = datastore.newKeyFactory().kind("Person");
+Key key = keyFactory.newKey("John Doe");
+Entity entity = Entity.builder(key)
+    .set("age", 30)
+    .set("favorite_food", "pizza")
+    .build();
+datastore.put(entity);
+```
+
+Later, if you want to get this entity back, add the following to your code:
+
+```java
+Entity johnEntity = datastore.get("John Doe");
+```
+
+#### Running a query
+In addition to retrieving entities by their keys, you can perform queries to retrieve entities by the values of their properties. A typical query includes an entity kind, filters to select entities with matching values, and sort orders to sequence the results. `gcloud-java-datastore` supports two types of queries: `StructuredQuery` (that allows you to construct query elements) and `GqlQuery` (which operates using [GQL syntax](https://cloud.google.com/datastore/docs/apis/gql/gql_reference)) in string format. In this tutorial, we will use a simple `StructuredQuery`.
+
+Suppose that you've added more people to Datastore, and now you want to find all people whose favorite food is pizza. Import the following:
+
+```java
+import com.google.gcloud.datastore.Query;
+import com.google.gcloud.datastore.QueryResults;
+import com.google.gcloud.datastore.StructuredQuery;
+import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
+```
+
+Then add the following code to your program:
+
+```java
+Query query = Query.entityQueryBuilder()
+    .kind("Person")
+    .filter(PropertyFilter.eq("favorite_food", "pizza"))
+    .build();
+QueryResults results = datastore.run(query);
+while (results.hasNext()) {
+  Entity currentEntity = results.next();
+}
+```
+
+#### Complete source code
+
+Here we put together all the code shown above into one program.  This program assumes that you are running on Compute Engine or from your own desktop. To run this example on App Engine, simply move the code from the main method to your application's servlet class.
+
+```java
+import com.google.gcloud.datastore.Datastore;
+import com.google.gcloud.datastore.DatastoreOptions;
+import com.google.gcloud.datastore.Entity;
+import com.google.gcloud.datastore.Key;
+import com.google.gcloud.datastore.KeyFactory;
+import com.google.gcloud.datastore.Query;
+import com.google.gcloud.datastore.QueryResults;
+import com.google.gcloud.datastore.StructuredQuery;
+import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
+
+public class GcloudJavaDatastoreExample {
+
+  public static void main(String[] args) {
+    // Create datastore service object.
+    // By default, credentials are inferred from the runtime environment.
+    Datastore datastore = DatastoreOptions.defaultInstance().service();
+
+    // Add an entity to Datastore
+    KeyFactory keyFactory = datastore.newKeyFactory().kind("Person");
+    Key key = keyFactory.newKey("John Doe");
+    Entity entity = Entity.builder(key)
+        .set("age", 51)
+        .set("favorite_food", "pizza")
+        .build();
+    datastore.put(entity);
+
+    // Get an entity from Datastore
+    Entity johnEntity = datastore.get(key);
+
+    // Add a couple more entities to make the query results more interesting
+    Key janeKey = keyFactory.newKey("Jane Doe");
+    Entity janeEntity = Entity.builder(janeKey)
+        .set("age", 44)
+        .set("favorite_food", "pizza")
+        .build();
+    Key joeKey = keyFactory.newKey("Joe Shmoe");
+    Entity joeEntity = Entity.builder(joeKey)
+        .set("age", 27)
+        .set("favorite_food", "sushi")
+        .build();
+    datastore.put(janeEntity, joeEntity);
+
+    // Run a query
+    Query query = Query.entityQueryBuilder()
+        .kind("Person")
+        .filter(PropertyFilter.eq("favorite_food", "pizza"))
+        .build();
+    QueryResults results = datastore.run(query);
+    while (results.hasNext()) {
+      Entity currentEntity = results.next();
+      // Do something using the entity.  (e.g. send an invite a pizza party)
+    }
+  }
 }
 ```
 

From 424f14450271fc3ffbe8095d791acd33888a8cb5 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Fri, 20 Nov 2015 08:35:55 -0800
Subject: [PATCH 61/94] fix snippets to match full code

---
 gcloud-java-datastore/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md
index 66cea4ef2037..e7a6cdd07781 100644
--- a/gcloud-java-datastore/README.md
+++ b/gcloud-java-datastore/README.md
@@ -91,7 +91,7 @@ Then add the following code to put an entity in Datastore.
 KeyFactory keyFactory = datastore.newKeyFactory().kind("Person");
 Key key = keyFactory.newKey("John Doe");
 Entity entity = Entity.builder(key)
-    .set("age", 30)
+    .set("age", 51)
     .set("favorite_food", "pizza")
     .build();
 datastore.put(entity);
@@ -100,7 +100,7 @@ datastore.put(entity);
 Later, if you want to get this entity back, add the following to your code:
 
 ```java
-Entity johnEntity = datastore.get("John Doe");
+Entity johnEntity = datastore.get(key);
 ```
 
 #### Running a query

From 791fba190e2d076468451ece01936deea8262236 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Fri, 20 Nov 2015 11:18:18 -0800
Subject: [PATCH 62/94] Add links to gcloud-common docs in READMEs

---
 README.md                       | 11 +++++++++--
 gcloud-java-core/README.md      | 10 +++++++++-
 gcloud-java-datastore/README.md | 10 +++++++++-
 gcloud-java-examples/README.md  | 10 +++++++++-
 gcloud-java-storage/README.md   | 10 +++++++++-
 gcloud-java/README.md           | 10 +++++++++-
 6 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 03bf2e69f915..e7bd4e77a002 100644
--- a/README.md
+++ b/README.md
@@ -75,7 +75,9 @@ Most `gcloud-java` libraries require a project ID.  There are multiple ways to s
 Authentication
 --------------
 
-There are multiple ways to authenticate to use Google Cloud services.
+First, ensure that the necessary Google Cloud APIs are enabled for your project. To do this, follow the instructions on the [authentication document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/authentication) shared by all the gcloud language libraries.
+
+Next, choose a method for authenticating API requests from within your project:
 
 1. When using `gcloud-java` libraries from within Compute/App Engine, no additional authentication steps are necessary.
 2. When using `gcloud-java` libraries elsewhere, there are two options:
@@ -180,6 +182,11 @@ if (blob == null) {
 }
 ```
 
+Troubleshooting
+---------------
+
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+
 Java Versions
 -------------
 
@@ -206,7 +213,7 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See [CONTRIBUTING] for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
 
 Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
diff --git a/gcloud-java-core/README.md b/gcloud-java-core/README.md
index 714c43f5c63d..4d6760d6259a 100644
--- a/gcloud-java-core/README.md
+++ b/gcloud-java-core/README.md
@@ -29,6 +29,11 @@ If you are using SBT, add this to your dependencies
 libraryDependencies += "com.google.gcloud" % "gcloud-java-core" % "0.0.12"
 ```
 
+Troubleshooting
+---------------
+
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+
 Java Versions
 -------------
 
@@ -39,7 +44,9 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See [CONTRIBUTING] for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+
+Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
 Versioning
 ----------
@@ -57,5 +64,6 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [cloud-platform]: https://cloud.google.com/
diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md
index e7a6cdd07781..57158e666769 100644
--- a/gcloud-java-datastore/README.md
+++ b/gcloud-java-datastore/README.md
@@ -189,6 +189,11 @@ public class GcloudJavaDatastoreExample {
 }
 ```
 
+Troubleshooting
+---------------
+
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+
 Java Versions
 -------------
 
@@ -215,7 +220,9 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See [CONTRIBUTING] for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+
+Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
 License
 -------
@@ -224,6 +231,7 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [TESTING]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/TESTING.md#testing-code-that-uses-datastore
 [cloud-platform]: https://cloud.google.com/
diff --git a/gcloud-java-examples/README.md b/gcloud-java-examples/README.md
index f381465676f5..05596546b78e 100644
--- a/gcloud-java-examples/README.md
+++ b/gcloud-java-examples/README.md
@@ -58,6 +58,11 @@ To run examples from your command line:
   $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" -Dexec.args="delete  test.txt"
 ```
 
+Troubleshooting
+---------------
+
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+
 Java Versions
 -------------
 
@@ -77,7 +82,9 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See [CONTRIBUTING] for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+
+Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
 License
 -------
@@ -86,6 +93,7 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [cloud-platform]: https://cloud.google.com/
 [developers-console]:https://console.developers.google.com/
diff --git a/gcloud-java-storage/README.md b/gcloud-java-storage/README.md
index 0302439a4314..dbd16b537413 100644
--- a/gcloud-java-storage/README.md
+++ b/gcloud-java-storage/README.md
@@ -82,6 +82,11 @@ if (!blob.exists()) {
 }
 ```
 
+Troubleshooting
+---------------
+
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+
 Java Versions
 -------------
 
@@ -108,7 +113,9 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See [CONTRIBUTING] for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+
+Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
 License
 -------
@@ -117,6 +124,7 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [TESTING]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/TESTING.md#testing-code-that-uses-storage
 [cloud-platform]: https://cloud.google.com/
diff --git a/gcloud-java/README.md b/gcloud-java/README.md
index 9a4ddfe77c83..ca3e8aa3d061 100644
--- a/gcloud-java/README.md
+++ b/gcloud-java/README.md
@@ -37,6 +37,11 @@ If you are using SBT, add this to your dependencies
 libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.12"
 ```
 
+Troubleshooting
+---------------
+
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+
 Java Versions
 -------------
 
@@ -56,7 +61,9 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See [CONTRIBUTING] for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+
+Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
 License
 -------
@@ -65,6 +72,7 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [cloud-platform]: https://cloud.google.com/
 [cloud-datastore]: https://cloud.google.com/datastore/docs

From 78f8f288f428618d4e5160407681adefd9c077e6 Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Fri, 20 Nov 2015 09:58:28 +0100
Subject: [PATCH 63/94] Limit batch deletes to 100, issue serveral batch if
 limit's exceeded

---
 .../google/gcloud/spi/DefaultStorageRpc.java  | 24 ++++++++-
 .../google/gcloud/storage/ITStorageTest.java  | 50 +++++++++++++++++++
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 4787b0888d7f..1e2549550544 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -35,7 +35,6 @@
 import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
 import com.google.api.client.googleapis.json.GoogleJsonError;
 import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.client.googleapis.media.MediaHttpDownloader;
 import com.google.api.client.http.ByteArrayContent;
 import com.google.api.client.http.GenericUrl;
 import com.google.api.client.http.HttpHeaders;
@@ -59,9 +58,10 @@
 import com.google.api.services.storage.model.Objects;
 import com.google.api.services.storage.model.StorageObject;
 import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.common.primitives.Ints;
 import com.google.gcloud.storage.StorageException;
 import com.google.gcloud.storage.StorageOptions;
 
@@ -69,6 +69,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -82,6 +83,7 @@ public class DefaultStorageRpc implements StorageRpc {
   // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
   private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 429, 408);
   private static final long MEGABYTE = 1024L * 1024L;
+  private static final int MAX_BATCH_DELETES = 100;
 
   public DefaultStorageRpc(StorageOptions options) {
     HttpTransport transport = options.httpTransportFactory().create();
@@ -361,6 +363,24 @@ public byte[] load(StorageObject from, Map options)
 
   @Override
   public BatchResponse batch(BatchRequest request) throws StorageException {
+    List>>> partitionedToDelete =
+        Lists.partition(request.toDelete, MAX_BATCH_DELETES);
+    Iterator>>> iterator = partitionedToDelete.iterator();
+    BatchRequest chunkRequest = new BatchRequest(iterator.hasNext() ? iterator.next() :
+        ImmutableList.>>of(), request.toUpdate, request.toGet);
+    BatchResponse response = batchChunk(chunkRequest);
+    Map> deletes =
+        Maps.newHashMapWithExpectedSize(request.toDelete.size());
+    deletes.putAll(response.deletes);
+    while (iterator.hasNext()) {
+      chunkRequest = new BatchRequest(iterator.next(), null, null);
+      BatchResponse deleteBatchResponse = batchChunk(chunkRequest);
+      deletes.putAll(deleteBatchResponse.deletes);
+    }
+    return new BatchResponse(deletes, response.updates, response.gets);
+  }
+
+  private BatchResponse batchChunk(BatchRequest request) {
     com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
     final Map> deletes =
         Maps.newConcurrentMap();
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
index 8bc3dfd5e207..ed58690efde7 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
@@ -25,6 +25,7 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import com.google.api.client.util.Lists;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.gcloud.Page;
@@ -65,6 +66,7 @@ public class ITStorageTest {
   private static final String CONTENT_TYPE = "text/plain";
   private static final byte[] BLOB_BYTE_CONTENT = {0xD, 0xE, 0xA, 0xD};
   private static final String BLOB_STRING_CONTENT = "Hello Google Cloud Storage!";
+  private static final int MAX_BATCH_DELETES = 100;
 
   @BeforeClass
   public static void beforeClass() {
@@ -623,6 +625,54 @@ public void testBatchRequest() {
     assertTrue(deleteResponse.deletes().get(1).get());
   }
 
+  @Test
+  public void testBatchRequestManyDeletes() {
+    List blobsToDelete = Lists.newArrayListWithCapacity(2 * MAX_BATCH_DELETES);
+    for (int i = 0; i < 2 * MAX_BATCH_DELETES; i++) {
+      blobsToDelete.add(BlobId.of(BUCKET, "test-batch-request-many-deletes-blob-" + i));
+    }
+    BatchRequest.Builder builder = BatchRequest.builder();
+    for (BlobId blob : blobsToDelete) {
+      builder.delete(blob);
+    }
+    String sourceBlobName1 = "test-batch-request-many-deletes-source-blob-1";
+    String sourceBlobName2 = "test-batch-request-many-deletes-source-blob-2";
+    BlobInfo sourceBlob1 = BlobInfo.builder(BUCKET, sourceBlobName1).build();
+    BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build();
+    assertNotNull(storage.create(sourceBlob1));
+    assertNotNull(storage.create(sourceBlob2));
+    BlobInfo updatedBlob2 = sourceBlob2.toBuilder().contentType(CONTENT_TYPE).build();
+
+    BatchRequest updateRequest = builder
+        .get(BUCKET, sourceBlobName1)
+        .update(updatedBlob2)
+        .build();
+    BatchResponse response = storage.apply(updateRequest);
+    assertEquals(2 * MAX_BATCH_DELETES, response.deletes().size());
+    assertEquals(1, response.updates().size());
+    assertEquals(1, response.gets().size());
+
+    // Check deletes
+    for (BatchResponse.Result deleteResult : response.deletes()) {
+      assertFalse(deleteResult.failed());
+      assertFalse(deleteResult.get());
+    }
+
+    // Check updates
+    BlobInfo remoteUpdatedBlob2 = response.updates().get(0).get();
+    assertEquals(sourceBlob2.bucket(), remoteUpdatedBlob2.bucket());
+    assertEquals(sourceBlob2.name(), remoteUpdatedBlob2.name());
+    assertEquals(updatedBlob2.contentType(), remoteUpdatedBlob2.contentType());
+
+    // Check gets
+    BlobInfo remoteBlob1 = response.gets().get(0).get();
+    assertEquals(sourceBlob1.bucket(), remoteBlob1.bucket());
+    assertEquals(sourceBlob1.name(), remoteBlob1.name());
+
+    assertTrue(storage.delete(BUCKET, sourceBlobName1));
+    assertTrue(storage.delete(BUCKET, sourceBlobName2));
+  }
+
   @Test
   public void testBatchRequestFail() {
     String blobName = "test-batch-request-blob-fail";

From 84c91bc838988fbd75e8dc8a1861f6c0abca542a Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Fri, 20 Nov 2015 16:19:59 -0800
Subject: [PATCH 64/94] fix link to auth readme

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e7bd4e77a002..319f5af161ba 100644
--- a/README.md
+++ b/README.md
@@ -75,7 +75,7 @@ Most `gcloud-java` libraries require a project ID.  There are multiple ways to s
 Authentication
 --------------
 
-First, ensure that the necessary Google Cloud APIs are enabled for your project. To do this, follow the instructions on the [authentication document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/authentication) shared by all the gcloud language libraries.
+First, ensure that the necessary Google Cloud APIs are enabled for your project. To do this, follow the instructions on the [authentication document](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/authentication/readme.md#authentication) shared by all the gcloud language libraries.
 
 Next, choose a method for authenticating API requests from within your project:
 

From 347d5bea76eb47f7f8233661a7c5f28aec28bd15 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Fri, 20 Nov 2015 17:07:29 -0800
Subject: [PATCH 65/94] use anchors in links

---
 README.md                       | 6 +++---
 gcloud-java-core/README.md      | 6 +++---
 gcloud-java-datastore/README.md | 6 +++---
 gcloud-java-examples/README.md  | 6 +++---
 gcloud-java-storage/README.md   | 7 ++++---
 gcloud-java/README.md           | 6 +++---
 6 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index 319f5af161ba..d045d292baac 100644
--- a/README.md
+++ b/README.md
@@ -185,7 +185,7 @@ if (blob == null) {
 Troubleshooting
 ---------------
 
-To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/troubleshooting/readme.md#troubleshooting).
 
 Java Versions
 -------------
@@ -213,7 +213,7 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/contributing/readme.md#how-to-contribute-to-gcloud) for more information on how to get started.
 
 Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
@@ -224,7 +224,7 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
-[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [TESTING]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/TESTING.md
 [cloud-platform]: https://cloud.google.com/
diff --git a/gcloud-java-core/README.md b/gcloud-java-core/README.md
index 4d6760d6259a..96b7a4e82021 100644
--- a/gcloud-java-core/README.md
+++ b/gcloud-java-core/README.md
@@ -32,7 +32,7 @@ libraryDependencies += "com.google.gcloud" % "gcloud-java-core" % "0.0.12"
 Troubleshooting
 ---------------
 
-To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/troubleshooting/readme.md#troubleshooting).
 
 Java Versions
 -------------
@@ -44,7 +44,7 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/contributing/readme.md#how-to-contribute-to-gcloud) for more information on how to get started.
 
 Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
@@ -64,6 +64,6 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
-[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [cloud-platform]: https://cloud.google.com/
diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md
index 57158e666769..de025d726721 100644
--- a/gcloud-java-datastore/README.md
+++ b/gcloud-java-datastore/README.md
@@ -192,7 +192,7 @@ public class GcloudJavaDatastoreExample {
 Troubleshooting
 ---------------
 
-To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/troubleshooting/readme.md#troubleshooting).
 
 Java Versions
 -------------
@@ -220,7 +220,7 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/contributing/readme.md#how-to-contribute-to-gcloud) for more information on how to get started.
 
 Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
@@ -231,7 +231,7 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
-[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [TESTING]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/TESTING.md#testing-code-that-uses-datastore
 [cloud-platform]: https://cloud.google.com/
diff --git a/gcloud-java-examples/README.md b/gcloud-java-examples/README.md
index 05596546b78e..2ccec3e81571 100644
--- a/gcloud-java-examples/README.md
+++ b/gcloud-java-examples/README.md
@@ -61,7 +61,7 @@ To run examples from your command line:
 Troubleshooting
 ---------------
 
-To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/troubleshooting/readme.md#troubleshooting).
 
 Java Versions
 -------------
@@ -82,7 +82,7 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/contributing/readme.md#how-to-contribute-to-gcloud) for more information on how to get started.
 
 Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
@@ -93,7 +93,7 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
-[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [cloud-platform]: https://cloud.google.com/
 [developers-console]:https://console.developers.google.com/
diff --git a/gcloud-java-storage/README.md b/gcloud-java-storage/README.md
index dbd16b537413..ea357fd81eaa 100644
--- a/gcloud-java-storage/README.md
+++ b/gcloud-java-storage/README.md
@@ -85,7 +85,7 @@ if (!blob.exists()) {
 Troubleshooting
 ---------------
 
-To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/troubleshooting/readme.md#troubleshooting).
 
 Java Versions
 -------------
@@ -113,7 +113,7 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/contributing/readme.md#how-to-contribute-to-gcloud) for more information on how to get started.
 
 Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
@@ -124,7 +124,7 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
-[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [TESTING]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/TESTING.md#testing-code-that-uses-storage
 [cloud-platform]: https://cloud.google.com/
@@ -133,3 +133,4 @@ Apache 2.0 - See [LICENSE] for more information.
 [cloud-storage-docs]: https://cloud.google.com/storage/docs/overview
 [cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets
 [storage-api]: http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/storage/package-summary.html
+[cloud-storage-activation]:https://cloud.google.com/storage/docs/signup?hl=en
diff --git a/gcloud-java/README.md b/gcloud-java/README.md
index ca3e8aa3d061..e381ca80cdaa 100644
--- a/gcloud-java/README.md
+++ b/gcloud-java/README.md
@@ -40,7 +40,7 @@ libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.12"
 Troubleshooting
 ---------------
 
-To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/troubleshooting).
+To get help, follow the `gcloud-java` links in the `gcloud-*` [shared Troubleshooting document](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/troubleshooting/readme.md#troubleshooting).
 
 Java Versions
 -------------
@@ -61,7 +61,7 @@ Contributing
 
 Contributions to this library are always welcome and highly encouraged.
 
-See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/tree/master/contributing) for more information on how to get started.
+See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` [shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/contributing/readme.md#how-to-contribute-to-gcloud) for more information on how to get started.
 
 Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information.
 
@@ -72,7 +72,7 @@ Apache 2.0 - See [LICENSE] for more information.
 
 
 [CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
-[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md
+[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct
 [LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
 [cloud-platform]: https://cloud.google.com/
 [cloud-datastore]: https://cloud.google.com/datastore/docs

From 08c0f28f6b05b77af79b042a0b56bdb21d2a2286 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Fri, 20 Nov 2015 16:18:45 -0800
Subject: [PATCH 66/94] Add step-by-step guide for storage README

---
 gcloud-java-storage/README.md | 163 ++++++++++++++++++++++++++++++----
 1 file changed, 147 insertions(+), 16 deletions(-)

diff --git a/gcloud-java-storage/README.md b/gcloud-java-storage/README.md
index 0302439a4314..43aca32fffc5 100644
--- a/gcloud-java-storage/README.md
+++ b/gcloud-java-storage/README.md
@@ -56,29 +56,160 @@ Cloud Storage for your project.
 See the ``gcloud-java`` API [storage documentation][storage-api] to learn how to interact
 with the Cloud Storage using this Client Library.
 
-Here is a code snippet showing a simple usage example from within Compute/App Engine.  Note that you must [supply credentials](https://github.com/GoogleCloudPlatform/gcloud-java#authentication) and a project ID if running this snippet elsewhere.
+Getting Started
+---------------
+#### Prerequisites
+For this tutorial, you will need a [Google Developers Console](https://console.developers.google.com/) project with the Storage JSON API enabled. You will need to [enable billing](https://support.google.com/cloud/answer/6158867?hl=en) to use Google Cloud Storage. [Follow these instructions](https://cloud.google.com/docs/authentication#preparation) to get your project set up. You will also need to set up the local development environment by [installing the Google Cloud SDK](https://cloud.google.com/sdk/) and running the following commands in command line: `gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`.
+
+#### Installation and setup
+You'll need to obtain the `gcloud-java-storage` library.  See the [Quickstart](#quickstart) section to add `gcloud-java-storage` as a dependency in your code.
+
+#### Creating an authorized service object
+To make authenticated requests to Google Cloud Storage, you must create a service object with credentials. You can then make API calls by calling methods on the Storage service object. The simplest way to authenticate is to use [Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials). These credentials are automatically inferred from your environment, so you only need the following code to create your service object:
+
+```java
+import com.google.gcloud.storage.Storage;
+import com.google.gcloud.storage.StorageOptions;
+
+Storage storage = StorageOptions.defaultInstance().service();
+```
+
+For other authentication options, see the [Authentication](https://github.com/GoogleCloudPlatform/gcloud-java#authentication) page.
+
+#### Storing data
+Stored objects are called "blobs" in `gcloud-java` and are organized into containers called "buckets".  In this code snippet, we will create a new bucket and upload a blob to that bucket.
+
+Add the following imports at the top of your file:
 
 ```java
 import static java.nio.charset.StandardCharsets.UTF_8;
 
+import com.google.gcloud.storage.BlobId;
+import com.google.gcloud.storage.BlobInfo;
+import com.google.gcloud.storage.BucketInfo;
+```
+
+Then add the following code to create a bucket and upload a simple blob.
+
+*Important: Bucket names have to be globally unique. If you choose a bucket name that already exists, you'll get a helpful error message telling you to choose another name. In the code below, replace "my_unique_bucket" with a unique bucket name.*
+
+```java
+// Create a bucket
+String bucketName = "my_unique_bucket"; // Remember to change this to something unique
+BucketInfo bucketInfo = storage.create(BucketInfo.builder(bucketName).build());
+
+// Upload a blob to the newly created bucket
+BlobId blobId = BlobId.of(bucketName, "my_blob_name");
+BlobInfo blobInfo = storage.create(BlobInfo.builder(blobId).build(),
+                                   "a simple blob".getBytes(UTF_8));
+```
+
+At this point, you will be able to see your newly created bucket and blob on the Google Developers Console.
+
+#### Retrieving data
+Now that we have content uploaded to the server, we can see how to read data from the server. Add the following import:
+
+```java
 import com.google.gcloud.storage.Blob;
-import com.google.gcloud.storage.Storage;
-import com.google.gcloud.storage.StorageOptions;
+```
 
-import java.nio.ByteBuffer;
-import java.nio.channels.WritableByteChannel;
+Then add the following lines to your program to get back the blob we uploaded.
 
-Storage storage = StorageOptions.defaultInstance().service();
-Blob blob = new Blob(storage, "bucket", "blob_name");
-if (!blob.exists()) {
-  storage2.create(blob.info(), "Hello, Cloud Storage!".getBytes(UTF_8));
-} else {
-  System.out.println("Updating content for " + blob.info().name());
-  byte[] prevContent = blob.content();
-  System.out.println(new String(prevContent, UTF_8));
-  WritableByteChannel channel = blob.writer();
-  channel.write(ByteBuffer.wrap("Updated content".getBytes(UTF_8)));
-  channel.close();
+```java
+Blob blob = Blob.load(storage, blobInfo.blobId());
+String blobContent = new String(blob.content(), UTF_8);
+```
+
+If others have permission to edit the blob, then you may want to call `reload` to get a more up to date copy of the blob later in your program. The following snippet shows how to get a new Blob object containing updated information.
+
+```
+Blob refreshedBlob = blob.reload();
+```
+
+#### Listing buckets and contents of buckets
+Suppose that you've added more buckets and blobs, and now you want to see the names of your buckets and the contents of each one. Add the following imports:
+
+```java
+import com.google.gcloud.storage.Bucket;
+
+import java.util.Iterator;
+```
+
+Then add the following code to list all your buckets and all the blobs inside your newly created bucket.
+
+```java
+// List all your buckets
+Iterator bucketInfoIterator = storage.list().iterateAll();
+System.out.println("My buckets:");
+while (bucketInfoIterator.hasNext()) {
+  System.out.println(bucketInfoIterator.next());
+}
+
+// List the blobs in a particular bucket
+Bucket bucket = Bucket.load(storage, bucketName);
+Iterator blobIterator = bucket.list().iterateAll();
+System.out.println("My blobs:");
+while (blobIterator.hasNext()) {
+  System.out.println(blobIterator.next().info());
+}
+```
+
+#### Complete source code
+
+Here we put together all the code shown above into one program.  This program assumes that you are running on Compute Engine or from your own desktop. To run this example on App Engine, simply move the code from the main method to your application's servlet class and change the print statements to display on your webpage.
+
+```java
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.gcloud.storage.Storage;
+import com.google.gcloud.storage.StorageOptions;
+import com.google.gcloud.storage.Blob;
+import com.google.gcloud.storage.BlobId;
+import com.google.gcloud.storage.BlobInfo;
+import com.google.gcloud.storage.Bucket;
+import com.google.gcloud.storage.BucketInfo;
+
+import java.util.Iterator;
+
+public class GcloudStorageExample {
+
+  public static void main(String[] args) {
+    // Create a service object
+    // Credentials are inferred from the environment.
+    Storage storage = StorageOptions.defaultInstance().service();
+
+    // Create a bucket
+    String bucketName = "my_unique_bucket-1323252"; // Remember to change this to something unique
+    BucketInfo bucketInfo = storage.create(BucketInfo.builder(bucketName).build());
+
+    // Upload a blob to the newly created bucket
+    BlobId blobId = BlobId.of(bucketName, "my_blob_name");
+    BlobInfo blobInfo = storage.create(BlobInfo.builder(blobId).build(),
+                                       "a simple blob".getBytes(UTF_8));
+
+    // Retrieve a blob from the server
+    Blob blob = Blob.load(storage, blobInfo.blobId());
+    String blobContent = new String(blob.content(), UTF_8);
+
+    // An example of how to get an updated blob
+    // This is useful in cases where others with access to your blob may have changed it.
+    Blob refreshedBlob = blob.reload();
+
+    // List all your buckets
+    Iterator bucketInfoIterator = storage.list().iterateAll();
+    System.out.println("My buckets:");
+    while (bucketInfoIterator.hasNext()) {
+      System.out.println(bucketInfoIterator.next());
+    }
+
+    // List the blobs in a particular bucket
+    Bucket bucket = Bucket.load(storage, bucketName);
+    Iterator blobIterator = bucket.list().iterateAll();
+    System.out.println("My blobs:");
+    while (blobIterator.hasNext()) {
+      System.out.println(blobIterator.next().info());
+    }
+  }
 }
 ```
 

From 3af2bbabb348c03a10fc643aa1f61b900f8e035b Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Mon, 23 Nov 2015 09:30:41 -0800
Subject: [PATCH 67/94] fix storage readme sample

---
 gcloud-java-storage/README.md | 60 +++++++++++------------------------
 1 file changed, 19 insertions(+), 41 deletions(-)

diff --git a/gcloud-java-storage/README.md b/gcloud-java-storage/README.md
index 43aca32fffc5..2ef75e9efb71 100644
--- a/gcloud-java-storage/README.md
+++ b/gcloud-java-storage/README.md
@@ -95,43 +95,29 @@ Then add the following code to create a bucket and upload a simple blob.
 
 ```java
 // Create a bucket
-String bucketName = "my_unique_bucket"; // Remember to change this to something unique
-BucketInfo bucketInfo = storage.create(BucketInfo.builder(bucketName).build());
+String bucketName = "my_unique_bucket"; // Change this to something unique
+BucketInfo bucketInfo = storage.create(BucketInfo.of(bucketName));
 
 // Upload a blob to the newly created bucket
 BlobId blobId = BlobId.of(bucketName, "my_blob_name");
-BlobInfo blobInfo = storage.create(BlobInfo.builder(blobId).build(),
-                                   "a simple blob".getBytes(UTF_8));
+BlobInfo blobInfo = storage.create(
+    BlobInfo.builder(blobId).contentType("text/plain").build(),
+    "a simple blob".getBytes(UTF_8));
 ```
 
 At this point, you will be able to see your newly created bucket and blob on the Google Developers Console.
 
 #### Retrieving data
-Now that we have content uploaded to the server, we can see how to read data from the server. Add the following import:
+Now that we have content uploaded to the server, we can see how to read data from the server.  Add the following line to your program to get back the blob we uploaded.
 
 ```java
-import com.google.gcloud.storage.Blob;
-```
-
-Then add the following lines to your program to get back the blob we uploaded.
-
-```java
-Blob blob = Blob.load(storage, blobInfo.blobId());
-String blobContent = new String(blob.content(), UTF_8);
-```
-
-If others have permission to edit the blob, then you may want to call `reload` to get a more up to date copy of the blob later in your program. The following snippet shows how to get a new Blob object containing updated information.
-
-```
-Blob refreshedBlob = blob.reload();
+String blobContent = new String(storage.readAllBytes(blobId), UTF_8);
 ```
 
 #### Listing buckets and contents of buckets
 Suppose that you've added more buckets and blobs, and now you want to see the names of your buckets and the contents of each one. Add the following imports:
 
 ```java
-import com.google.gcloud.storage.Bucket;
-
 import java.util.Iterator;
 ```
 
@@ -146,11 +132,10 @@ while (bucketInfoIterator.hasNext()) {
 }
 
 // List the blobs in a particular bucket
-Bucket bucket = Bucket.load(storage, bucketName);
-Iterator blobIterator = bucket.list().iterateAll();
+Iterator blobIterator = storage.list(bucketName).iterateAll();
 System.out.println("My blobs:");
 while (blobIterator.hasNext()) {
-  System.out.println(blobIterator.next().info());
+  System.out.println(blobIterator.next());
 }
 ```
 
@@ -161,13 +146,11 @@ Here we put together all the code shown above into one program.  This program as
 ```java
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import com.google.gcloud.storage.Storage;
-import com.google.gcloud.storage.StorageOptions;
-import com.google.gcloud.storage.Blob;
 import com.google.gcloud.storage.BlobId;
 import com.google.gcloud.storage.BlobInfo;
-import com.google.gcloud.storage.Bucket;
 import com.google.gcloud.storage.BucketInfo;
+import com.google.gcloud.storage.Storage;
+import com.google.gcloud.storage.StorageOptions;
 
 import java.util.Iterator;
 
@@ -179,21 +162,17 @@ public class GcloudStorageExample {
     Storage storage = StorageOptions.defaultInstance().service();
 
     // Create a bucket
-    String bucketName = "my_unique_bucket-1323252"; // Remember to change this to something unique
-    BucketInfo bucketInfo = storage.create(BucketInfo.builder(bucketName).build());
+    String bucketName = "my_unique_bucket"; // Change this to something unique
+    BucketInfo bucketInfo = storage.create(BucketInfo.of(bucketName));
 
     // Upload a blob to the newly created bucket
     BlobId blobId = BlobId.of(bucketName, "my_blob_name");
-    BlobInfo blobInfo = storage.create(BlobInfo.builder(blobId).build(),
-                                       "a simple blob".getBytes(UTF_8));
+    BlobInfo blobInfo = storage.create(
+        BlobInfo.builder(blobId).contentType("text/plain").build(),
+        "a simple blob".getBytes(UTF_8));
 
     // Retrieve a blob from the server
-    Blob blob = Blob.load(storage, blobInfo.blobId());
-    String blobContent = new String(blob.content(), UTF_8);
-
-    // An example of how to get an updated blob
-    // This is useful in cases where others with access to your blob may have changed it.
-    Blob refreshedBlob = blob.reload();
+    String blobContent = new String(storage.readAllBytes(blobId), UTF_8);
 
     // List all your buckets
     Iterator bucketInfoIterator = storage.list().iterateAll();
@@ -203,11 +182,10 @@ public class GcloudStorageExample {
     }
 
     // List the blobs in a particular bucket
-    Bucket bucket = Bucket.load(storage, bucketName);
-    Iterator blobIterator = bucket.list().iterateAll();
+    Iterator blobIterator = storage.list(bucketName).iterateAll();
     System.out.println("My blobs:");
     while (blobIterator.hasNext()) {
-      System.out.println(blobIterator.next().info());
+      System.out.println(blobIterator.next());
     }
   }
 }

From 8ebfdf87356e9f0df3921e404b5bcafb7705b5ed Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Mon, 23 Nov 2015 11:09:14 -0800
Subject: [PATCH 68/94] add link to bucket naming rules

---
 gcloud-java-storage/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gcloud-java-storage/README.md b/gcloud-java-storage/README.md
index 2ef75e9efb71..1053f096eeb5 100644
--- a/gcloud-java-storage/README.md
+++ b/gcloud-java-storage/README.md
@@ -91,7 +91,7 @@ import com.google.gcloud.storage.BucketInfo;
 
 Then add the following code to create a bucket and upload a simple blob.
 
-*Important: Bucket names have to be globally unique. If you choose a bucket name that already exists, you'll get a helpful error message telling you to choose another name. In the code below, replace "my_unique_bucket" with a unique bucket name.*
+*Important: Bucket names have to be globally unique. If you choose a bucket name that already exists, you'll get a helpful error message telling you to choose another name. In the code below, replace "my_unique_bucket" with a unique bucket name. See more about naming rules [here](https://cloud.google.com/storage/docs/bucket-naming?hl=en#requirements).*
 
 ```java
 // Create a bucket

From 539ac61bfc31a544bb6ebf5a29d680bbaf618110 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Mon, 23 Nov 2015 13:09:42 -0800
Subject: [PATCH 69/94] Add back AppEngineCredentials

---
 README.md                                     |  6 +--
 gcloud-java-core/pom.xml                      | 11 -----
 .../com/google/gcloud/AuthCredentials.java    | 46 ++++++++++++++++++-
 .../com/google/gcloud/ServiceOptions.java     |  9 ++++
 .../gcloud/datastore/SerializationTest.java   |  1 +
 .../gcloud/storage/SerializationTest.java     |  1 +
 6 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index ccdd18c31052..c697796f0a4f 100644
--- a/README.md
+++ b/README.md
@@ -96,9 +96,9 @@ There are multiple ways to authenticate to use Google Cloud services.
 `gcloud-java` looks for credentials in the following order, stopping once it finds credentials:
 
 1. Credentials supplied when building the service options
-2. Key file pointed to by the GOOGLE_APPLICATION_CREDENTIALS environment variable
-3. Google Cloud SDK credentials
-4. App Engine credentials
+2. App Engine credentials
+3. Key file pointed to by the GOOGLE_APPLICATION_CREDENTIALS environment variable
+4. Google Cloud SDK credentials
 5. Compute Engine credentials
 
 Google Cloud Datastore
diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml
index e2ac0ddd81b6..c0248004d335 100644
--- a/gcloud-java-core/pom.xml
+++ b/gcloud-java-core/pom.xml
@@ -33,17 +33,6 @@
         
       
     
-    
-      com.google.auth
-      google-auth-library-appengine
-      0.3.1
-      
-        
-          com.google.guava
-          guava-jdk5
-        
-      
-    
     
       com.google.http-client
       google-http-client
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
index 09cdcc16f7de..3303e4f8a652 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -19,6 +19,7 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential;
 import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.json.jackson.JacksonFactory;
@@ -38,6 +39,45 @@
  */
 public abstract class AuthCredentials implements Restorable {
 
+  private static class AppEngineAuthCredentials extends AuthCredentials {
+
+    private static final AuthCredentials INSTANCE = new AppEngineAuthCredentials();
+    private static final AppEngineAuthCredentialsState STATE =
+        new AppEngineAuthCredentialsState();
+
+    private static class AppEngineAuthCredentialsState
+        implements RestorableState, Serializable {
+
+      private static final long serialVersionUID = 3558563960848658928L;
+
+      @Override
+      public AuthCredentials restore() {
+        return INSTANCE;
+      }
+
+      @Override
+      public int hashCode() {
+        return getClass().getName().hashCode();
+      }
+
+      @Override
+      public boolean equals(Object obj) {
+        return obj instanceof AppEngineAuthCredentialsState;
+      }
+    }
+
+    @Override
+    protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
+        Set scopes) {
+      return new AppIdentityCredential(scopes);
+    }
+
+    @Override
+    public RestorableState capture() {
+      return STATE;
+    }
+  }
+
   public static class ServiceAccountAuthCredentials extends AuthCredentials {
 
     private final String account;
@@ -181,12 +221,16 @@ public RestorableState capture() {
   protected abstract HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
       Set scopes);
 
+  public static AuthCredentials createForAppEngine() {
+    return AppEngineAuthCredentials.INSTANCE;
+  }
+
   /**
    * Returns the Application Default Credentials.
    *
    * 
Returns the Application Default Credentials which are credentials that identify and
    * authorize the whole application. This is the built-in service account if running on
-   * Google App/Compute Engine or the credentials file can be read from the path in the environment
+   * Google Compute Engine or the credentials file can be read from the path in the environment
    * variable GOOGLE_APPLICATION_CREDENTIALS.
    * 

    *
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
index e5dba231d3ff..a5601e35a6db 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -356,6 +356,15 @@ protected boolean projectIdRequired() {
   }
 
   private static AuthCredentials defaultAuthCredentials() {
+    // Consider App Engine.
+    if (appEngineAppId() != null) {
+      try {
+        return AuthCredentials.createForAppEngine();
+      } catch (Exception ignore) {
+        // Maybe not on App Engine
+      }
+    }
+
     try {
       return AuthCredentials.createApplicationDefaults();
     } catch (Exception ex) {
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 8fba8de8866b..32e14fb47ea0 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -134,6 +134,7 @@ public class SerializationTest {
   @Test
   public void testServiceOptions() throws Exception {
     DatastoreOptions options = DatastoreOptions.builder()
+        .authCredentials(AuthCredentials.createForAppEngine())
         .normalizeDataset(false)
         .projectId("ds1")
         .build();
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
index a1b647fbba07..2d80191aeb2d 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -75,6 +75,7 @@ public class SerializationTest {
   public void testServiceOptions() throws Exception {
     StorageOptions options = StorageOptions.builder()
         .projectId("p1")
+        .authCredentials(AuthCredentials.createForAppEngine())
         .build();
     StorageOptions serializedCopy = serializeAndDeserialize(options);
     assertEquals(options, serializedCopy);

From fc2ab109391b3c3f7fd6caeeed71e27f2c0d4692 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Mon, 23 Nov 2015 19:11:08 -0800
Subject: [PATCH 70/94] Avoid throwing exception when checking isLocalHost in
 Datastore

---
 .../main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java  | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
index ccb89267a29e..4b39f7a3c4be 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
@@ -42,9 +42,7 @@
 import org.json.JSONTokener;
 
 import java.net.InetAddress;
-import java.net.MalformedURLException;
 import java.net.URL;
-import java.net.UnknownHostException;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -95,7 +93,7 @@ private static boolean isLocalHost(String host) {
         }
         InetAddress hostAddr = InetAddress.getByName(new URL(normalizedHost).getHost());
         return hostAddr.isAnyLocalAddress() || hostAddr.isLoopbackAddress();
-      } catch (UnknownHostException | MalformedURLException e) {
+      } catch (Exception e) {
         // ignore
       }
     }

From 8808c54c09a1d80060d8ea804164dc26ee4ce229 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Mon, 23 Nov 2015 19:31:43 -0800
Subject: [PATCH 71/94] Add link about datastore indexing in docs

---
 gcloud-java-datastore/README.md                               | 2 ++
 .../src/main/java/com/google/gcloud/datastore/Query.java      | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md
index de025d726721..aeaae2b621c6 100644
--- a/gcloud-java-datastore/README.md
+++ b/gcloud-java-datastore/README.md
@@ -128,6 +128,8 @@ while (results.hasNext()) {
 }
 ```
 
+Cloud Datastore relies on indexing to run queries. Indexing is turned on by default for most types of properties. To read more about indexing, see the [Cloud Datastore Index Configuration documentation](https://cloud.google.com/datastore/docs/tools/indexconfig).
+
 #### Complete source code
 
 Here we put together all the code shown above into one program.  This program assumes that you are running on Compute Engine or from your own desktop. To run this example on App Engine, simply move the code from the main method to your application's servlet class.
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java
index 343535d94628..5791d37e9426 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java
@@ -35,6 +35,10 @@
  * A Google Cloud Datastore query.
  * For usage examples see {@link GqlQuery} and {@link StructuredQuery}.
  *
+ * Note that queries require proper indexing. See
+ * 
+ * Cloud Datastore Index Configuration for help configuring indexes.
+ *
  * @param  the type of the values returned by this query.
  * @see Datastore Queries
  */

From 0a05851b3ad9d6138f57c726ea888d4675ef375b Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Tue, 24 Nov 2015 15:38:54 +0100
Subject: [PATCH 72/94] Translate SocketTimeoutException to retryable service
 exception

---
 .../com/google/gcloud/spi/DefaultDatastoreRpc.java  | 13 ++++++++++---
 .../com/google/gcloud/spi/DefaultStorageRpc.java    |  7 ++++++-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
index 4b39f7a3c4be..fbe3bff780da 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
@@ -42,6 +42,7 @@
 import org.json.JSONTokener;
 
 import java.net.InetAddress;
+import java.net.SocketTimeoutException;
 import java.net.URL;
 import java.util.HashMap;
 import java.util.Map;
@@ -121,9 +122,15 @@ private static DatastoreRpcException translate(DatastoreException exception) {
     if (reason == null) {
       reason = HTTP_STATUS_TO_REASON.get(exception.getCode());
     }
-    return reason != null
-        ? new DatastoreRpcException(reason)
-        : new DatastoreRpcException("Unknown", exception.getCode(), false, message);
+    if (reason != null) {
+      return new DatastoreRpcException(reason);
+    } else {
+      boolean retryable = false;
+      if (exception.getCause() instanceof SocketTimeoutException) {
+        retryable = true;
+      }
+      return new DatastoreRpcException("Unknown", exception.getCode(), retryable, message);
+    }
   }
 
   @Override
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 1e2549550544..d874f99ebb4c 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -68,6 +68,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.SocketTimeoutException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -101,7 +102,11 @@ private static StorageException translate(IOException exception) {
         && ((GoogleJsonResponseException) exception).getDetails() != null) {
       translated = translate(((GoogleJsonResponseException) exception).getDetails());
     } else {
-      translated = new StorageException(0, exception.getMessage(), false);
+      boolean retryable = false;
+      if (exception instanceof SocketTimeoutException) {
+        retryable = true;
+      }
+      translated = new StorageException(0, exception.getMessage(), retryable);
     }
     translated.initCause(exception);
     return translated;

From a0f2d206477129511fde909ae57a9ce46e8858cf Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Tue, 24 Nov 2015 19:01:55 +0100
Subject: [PATCH 73/94] Set special reason for timeout exceptions

---
 .../main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
index fbe3bff780da..028027f4bc33 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
@@ -126,10 +126,12 @@ private static DatastoreRpcException translate(DatastoreException exception) {
       return new DatastoreRpcException(reason);
     } else {
       boolean retryable = false;
+      reasonStr = "Unknown";
       if (exception.getCause() instanceof SocketTimeoutException) {
         retryable = true;
+        reasonStr = "Request timeout";
       }
-      return new DatastoreRpcException("Unknown", exception.getCode(), retryable, message);
+      return new DatastoreRpcException(reasonStr, exception.getCode(), retryable, message);
     }
   }
 

From 1a6dcde82e01bc4432e14675b39bffddae97c8b6 Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Tue, 24 Nov 2015 19:26:48 +0100
Subject: [PATCH 74/94] Fix toBuilder so that
 info.equals(info.toBuilder().build()) is true

---
 .../com/google/gcloud/storage/BlobInfo.java   | 47 ++++++++++---------
 .../com/google/gcloud/storage/BucketInfo.java | 45 +++++++++---------
 .../google/gcloud/storage/BlobInfoTest.java   |  6 +++
 .../google/gcloud/storage/BucketInfoTest.java |  2 +-
 4 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java
index 29cf16dcd617..9d1fd4f5e25c 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java
@@ -106,7 +106,7 @@ public static final class Builder {
     private String contentLanguage;
     private Integer componentCount;
     private String cacheControl;
-    private ImmutableList acl;
+    private List acl;
     private Acl.Entity owner;
     private Long size;
     private String etag;
@@ -121,6 +121,29 @@ public static final class Builder {
 
     private Builder() {}
 
+    private Builder(BlobInfo blobInfo) {
+      blobId = blobInfo.blobId;
+      id = blobInfo.id;
+      cacheControl = blobInfo.cacheControl;
+      contentEncoding = blobInfo.contentEncoding;
+      contentType = blobInfo.contentType;
+      contentDisposition = blobInfo.contentDisposition;
+      contentLanguage = blobInfo.contentLanguage;
+      componentCount = blobInfo.componentCount;
+      acl = blobInfo.acl;
+      owner = blobInfo.owner;
+      size = blobInfo.size;
+      etag = blobInfo.etag;
+      selfLink = blobInfo.selfLink;
+      md5 = blobInfo.md5;
+      crc32c = blobInfo.crc32c;
+      mediaLink = blobInfo.mediaLink;
+      metadata = blobInfo.metadata;
+      metageneration = blobInfo.metageneration;
+      deleteTime = blobInfo.deleteTime;
+      updateTime = blobInfo.updateTime;
+    }
+
     /**
      * Sets the blob identity.
      */
@@ -503,27 +526,7 @@ public Long updateTime() {
    * Returns a builder for the current blob.
    */
   public Builder toBuilder() {
-    return new Builder()
-        .blobId(blobId)
-        .id(id)
-        .cacheControl(cacheControl)
-        .contentEncoding(contentEncoding)
-        .contentType(contentType)
-        .contentDisposition(contentDisposition)
-        .contentLanguage(contentLanguage)
-        .componentCount(componentCount)
-        .crc32c(crc32c)
-        .md5(md5)
-        .deleteTime(deleteTime)
-        .updateTime(updateTime)
-        .mediaLink(mediaLink)
-        .metadata(metadata)
-        .metageneration(metageneration)
-        .acl(acl)
-        .owner(owner)
-        .size(size)
-        .etag(etag)
-        .selfLink(selfLink);
+    return new Builder(this);
   }
 
   @Override
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
index d5a382446709..a8cc4a0f32d8 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -325,18 +325,37 @@ public static final class Builder {
     private Boolean versioningEnabled;
     private String indexPage;
     private String notFoundPage;
-    private ImmutableList deleteRules;
+    private List deleteRules;
     private String storageClass;
     private String location;
     private String etag;
     private Long createTime;
     private Long metageneration;
-    private ImmutableList cors;
-    private ImmutableList acl;
-    private ImmutableList defaultAcl;
+    private List cors;
+    private List acl;
+    private List defaultAcl;
 
     private Builder() {}
 
+    private Builder(BucketInfo bucketInfo) {
+      id = bucketInfo.id;
+      name = bucketInfo.name;
+      etag = bucketInfo.etag;
+      createTime = bucketInfo.createTime;
+      metageneration = bucketInfo.metageneration;
+      location = bucketInfo.location;
+      storageClass = bucketInfo.storageClass;
+      cors = bucketInfo.cors;
+      acl = bucketInfo.acl;
+      defaultAcl = bucketInfo.defaultAcl;
+      owner = bucketInfo.owner;
+      selfLink = bucketInfo.selfLink;
+      versioningEnabled = bucketInfo.versioningEnabled;
+      indexPage = bucketInfo.indexPage;
+      notFoundPage = bucketInfo.notFoundPage;
+      deleteRules = bucketInfo.deleteRules;
+    }
+
     /**
      * Sets the bucket's name.
      */
@@ -629,23 +648,7 @@ public List defaultAcl() {
    * Returns a builder for the current bucket.
    */
   public Builder toBuilder() {
-    return new Builder()
-        .name(name)
-        .id(id)
-        .createTime(createTime)
-        .etag(etag)
-        .metageneration(metageneration)
-        .cors(cors)
-        .acl(acl)
-        .defaultAcl(defaultAcl)
-        .location(location)
-        .storageClass(storageClass)
-        .owner(owner)
-        .selfLink(selfLink)
-        .versioningEnabled(versioningEnabled)
-        .indexPage(indexPage)
-        .notFoundPage(notFoundPage)
-        .deleteRules(deleteRules);
+    return new Builder(this);
   }
 
   @Override
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java
index 7214170afe9a..36b027dc7278 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java
@@ -88,6 +88,12 @@ public void testToBuilder() {
     compareBlobs(BLOB_INFO, blobInfo);
   }
 
+  @Test
+  public void testToBuilderIncomplete() {
+    BlobInfo incompleteBlobInfo = BlobInfo.builder(BlobId.of("b2", "n2")).build();
+    compareBlobs(incompleteBlobInfo, incompleteBlobInfo.toBuilder().build());
+  }
+
   @Test
   public void testBuilder() {
     assertEquals("b", BLOB_INFO.bucket());
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java
index 4fa420b4b6e1..e0de2b77a899 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java
@@ -91,7 +91,7 @@ public void testToBuilder() {
   @Test
   public void testToBuilderIncomplete() {
     BucketInfo incompleteBucketInfo = BucketInfo.builder("b").build();
-    assertEquals(incompleteBucketInfo.name(), incompleteBucketInfo.toBuilder().build().name());
+    compareBuckets(incompleteBucketInfo, incompleteBucketInfo.toBuilder().build());
   }
 
   @Test

From 2ef47e869450d14d4a1153bcbff17e1c9c36ea60 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Tue, 24 Nov 2015 13:20:16 -0800
Subject: [PATCH 75/94] Use NetHttpTransport when running on Compute

---
 .../com/google/gcloud/ServiceOptions.java     | 20 -------------------
 1 file changed, 20 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
index a5601e35a6db..25fda29c363d 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -21,13 +21,10 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.client.extensions.appengine.http.UrlFetchTransport;
-import com.google.api.client.googleapis.compute.ComputeCredential;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
 import com.google.api.client.http.HttpRequest;
 import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
-import com.google.api.client.json.jackson.JacksonFactory;
 import com.google.common.collect.Iterables;
 import com.google.gcloud.spi.ServiceRpcFactory;
 
@@ -44,7 +41,6 @@
 import java.lang.reflect.Method;
 import java.net.HttpURLConnection;
 import java.net.URL;
-import java.security.GeneralSecurityException;
 import java.util.Enumeration;
 import java.util.Locale;
 import java.util.Objects;
@@ -113,24 +109,8 @@ public HttpTransport create() {
           // Maybe not on App Engine
         }
       }
-      // Consider Compute
-      try {
-        return getComputeHttpTransport();
-      } catch (Exception e) {
-        // Maybe not on GCE
-      }
       return new NetHttpTransport();
     }
-
-    private static HttpTransport getComputeHttpTransport()
-        throws IOException, GeneralSecurityException {
-      NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
-      // Try to connect using Google Compute Engine service account credentials.
-      ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory());
-      // Force token refresh to detect if we are running on Google Compute Engine.
-      credential.refreshToken();
-      return transport;
-    }
   }
 
   /**

From f87a394379e3ecb662370da5dc5e2b0ddfa4a91f Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Tue, 24 Nov 2015 18:40:22 -0800
Subject: [PATCH 76/94] Fix dependency on two different versions of the
 GoogleCredential class

---
 gcloud-java-datastore/pom.xml | 6 ++++++
 gcloud-java-storage/pom.xml   | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/gcloud-java-datastore/pom.xml b/gcloud-java-datastore/pom.xml
index f2743c8e2af1..4bf5caab2ff8 100644
--- a/gcloud-java-datastore/pom.xml
+++ b/gcloud-java-datastore/pom.xml
@@ -27,6 +27,12 @@
       google-api-services-datastore-protobuf
       v1beta2-rev1-2.1.2
       compile
+      
+        
+          com.google.api-client
+          google-api-client
+        
+      
     
     
       junit
diff --git a/gcloud-java-storage/pom.xml b/gcloud-java-storage/pom.xml
index b592200243ef..3e0b6a9490a8 100644
--- a/gcloud-java-storage/pom.xml
+++ b/gcloud-java-storage/pom.xml
@@ -32,6 +32,10 @@
 	  com.google.guava
 	  guava-jdk5
 	
+        
+          com.google.api-client
+          google-api-client
+        
       
     
     

From 18302758569d6a2a094a5a7986e2b10904f48e7a Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Mon, 30 Nov 2015 16:58:33 +0100
Subject: [PATCH 77/94] Enable travis to branches other than master

---
 .travis.yml         | 3 ---
 utilities/verify.sh | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ab421366db6a..c023116917a3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,9 +10,6 @@ before_install:
 install: mvn install -DskipTests=true -Dgpg.skip=true
 script: 
 - utilities/verify.sh
-branches:
-  only:
-  - master
 after_success:
 - utilities/after_success.sh
 env:
diff --git a/utilities/verify.sh b/utilities/verify.sh
index 463180415e98..d98e4ab53513 100755
--- a/utilities/verify.sh
+++ b/utilities/verify.sh
@@ -4,7 +4,7 @@ source ./utilities/integration_test_env.sh
 # This script is used by Travis-CI to run tests.
 # This script is referenced in .travis.yml.
 
-if [ "${TRAVIS_BRANCH}" == "master" -a "${TRAVIS_PULL_REQUEST}" == "false" ]; then
+if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
     # Get signing tools and API keyfile
     openssl aes-256-cbc -K $encrypted_631490ecae8f_key -iv $encrypted_631490ecae8f_iv -in target/travis/signing-tools.tar.enc -out $TRAVIS_BUILD_DIR/signing-tools.tar -d
     mkdir $TRAVIS_BUILD_DIR/signing-tools

From 57d6e3f6eac16397fdef7ffc30b022b4faaf2a24 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Mon, 30 Nov 2015 20:29:57 -0800
Subject: [PATCH 78/94] Remove dependency on http initializer in
 AuthCredentials and remove dependency on
 com.google.api.client.googleapis.auth.oauth2.GoogleCredential

---
 .../com/google/gcloud/AuthCredentials.java    | 114 +++++++++++++-----
 .../com/google/gcloud/ServiceOptions.java     |   4 +-
 2 files changed, 85 insertions(+), 33 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
index 3303e4f8a652..779a60fb8a1b 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -18,21 +18,17 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential;
-import com.google.api.client.http.HttpRequestInitializer;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.json.jackson.JacksonFactory;
-import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.AccessToken;
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.auth.oauth2.ServiceAccountCredentials;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Serializable;
+import java.lang.reflect.Method;
 import java.security.PrivateKey;
+import java.util.Collection;
 import java.util.Objects;
-import java.util.Set;
 
 /**
  * Credentials for accessing Google Cloud services.
@@ -42,8 +38,66 @@ public abstract class AuthCredentials implements Restorable {
   private static class AppEngineAuthCredentials extends AuthCredentials {
 
     private static final AuthCredentials INSTANCE = new AppEngineAuthCredentials();
-    private static final AppEngineAuthCredentialsState STATE =
-        new AppEngineAuthCredentialsState();
+    private static final AppEngineAuthCredentialsState STATE = new AppEngineAuthCredentialsState();
+
+    private static class AppEngineCredentials extends GoogleCredentials {
+
+      private final Object appIdentityService;
+      private final Collection scopes;
+      private final boolean scopesRequired;
+
+      AppEngineCredentials() {
+        try {
+          Class factoryClass =
+              Class.forName("com.google.appengine.api.appidentity.AppIdentityServiceFactory");
+          Method method = factoryClass.getMethod("getAppIdentityService");
+          this.appIdentityService = method.invoke(null);
+          this.scopes = null;
+          this.scopesRequired = true;
+        } catch (Exception e) {
+          throw new RuntimeException("Could not create AppEngineCredentials using reflection.");
+        }
+      }
+
+      AppEngineCredentials(Collection scopes, Object appIdentityService) {
+          this.appIdentityService = appIdentityService;
+          this.scopes = scopes;
+          this.scopesRequired = (scopes == null || scopes.isEmpty());
+      }
+
+      /**
+       * Refresh the access token by getting it from the App Identity service
+       */
+      @Override
+      public AccessToken refreshAccessToken() throws IOException {
+        if (createScopedRequired()) {
+          throw new IOException("AppEngineCredentials requires createScoped call before use.");
+        }
+        try {
+          Class serviceClass =
+              Class.forName("com.google.appengine.api.appidentity.AppIdentityService");
+          Class tokenResultClass = Class.forName(
+              "com.google.appengine.api.appidentity.AppIdentityService$GetAccessTokenResult");
+          Method getAccessTokenResult = serviceClass.getMethod("getAccessToken", Iterable.class);
+          Object accessTokenResult = getAccessTokenResult.invoke(appIdentityService, scopes);
+          Method getAccessToken = tokenResultClass.getMethod("getAccessToken");
+          String accessToken = (String) getAccessToken.invoke(accessTokenResult);
+          return new AccessToken(accessToken, null);
+        } catch (Exception e) {
+          throw new RuntimeException("Could not get the access token using reflection.");
+        }
+      }
+
+      @Override
+      public boolean createScopedRequired() {
+        return scopesRequired;
+      }
+
+      @Override
+      public GoogleCredentials createScoped(Collection scopes) {
+        return new AppEngineCredentials(scopes, appIdentityService);
+      }
+    }
 
     private static class AppEngineAuthCredentialsState
         implements RestorableState, Serializable {
@@ -67,9 +121,8 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
-        Set scopes) {
-      return new AppIdentityCredential(scopes);
+    protected GoogleCredentials credentials() {
+      return new AppEngineCredentials();
     }
 
     @Override
@@ -133,17 +186,10 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    protected HttpRequestInitializer httpRequestInitializer(
-        HttpTransport transport, Set scopes) {
-      GoogleCredential.Builder builder = new GoogleCredential.Builder()
-          .setTransport(transport)
-          .setJsonFactory(new JacksonFactory());
-      if (privateKey != null) {
-        builder.setServiceAccountPrivateKey(privateKey);
-        builder.setServiceAccountId(account);
-        builder.setServiceAccountScopes(scopes);
-      }
-      return builder.build();
+    protected GoogleCredentials credentials() {
+      return (privateKey == null)
+          ? new GoogleCredentials(null)
+          : new ServiceAccountCredentials(null, account, privateKey, null, null);
     }
 
     public String account() {
@@ -198,9 +244,8 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
-        Set scopes) {
-      return new HttpCredentialsAdapter(googleCredentials.createScoped(scopes));
+    protected GoogleCredentials credentials() {
+      return googleCredentials;
     }
 
     public ServiceAccountAuthCredentials toServiceAccountCredentials() {
@@ -218,8 +263,7 @@ public RestorableState capture() {
     }
   }
 
-  protected abstract HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
-      Set scopes);
+  protected abstract GoogleCredentials credentials();
 
   public static AuthCredentials createForAppEngine() {
     return AppEngineAuthCredentials.INSTANCE;
@@ -271,9 +315,17 @@ public static ServiceAccountAuthCredentials createFor(String account, PrivateKey
    */
   public static ServiceAccountAuthCredentials createForJson(InputStream jsonCredentialStream)
       throws IOException {
-    GoogleCredential tempCredentials = GoogleCredential.fromStream(jsonCredentialStream);
-    return new ServiceAccountAuthCredentials(tempCredentials.getServiceAccountId(),
-        tempCredentials.getServiceAccountPrivateKey());
+    GoogleCredentials tempCredentials = GoogleCredentials.fromStream(jsonCredentialStream);
+    if (tempCredentials instanceof ServiceAccountCredentials) {
+      ServiceAccountCredentials tempServiceAccountCredentials =
+          (ServiceAccountCredentials) tempCredentials;
+      return new ServiceAccountAuthCredentials(
+          tempServiceAccountCredentials.getClientEmail(),
+          tempServiceAccountCredentials.getPrivateKey());
+    } else {
+      throw new IOException(
+          "The given JSON Credentials Stream is not a service account credential.");
+    }
   }
 
   public static AuthCredentials noCredentials() {
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
index 25fda29c363d..bc0e91d0a694 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -25,6 +25,7 @@
 import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.auth.http.HttpCredentialsAdapter;
 import com.google.common.collect.Iterables;
 import com.google.gcloud.spi.ServiceRpcFactory;
 
@@ -508,9 +509,8 @@ public RetryParams retryParams() {
    * options.
    */
   public HttpRequestInitializer httpRequestInitializer() {
-    HttpTransport httpTransport = httpTransportFactory.create();
     final HttpRequestInitializer baseRequestInitializer =
-        authCredentials().httpRequestInitializer(httpTransport, scopes());
+        new HttpCredentialsAdapter(authCredentials().credentials().createScoped(scopes()));
     return new HttpRequestInitializer() {
       @Override
       public void initialize(HttpRequest httpRequest) throws IOException {

From b5c1caec2a3d1cfe5a0b7a118154b3e333471b6f Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Tue, 1 Dec 2015 09:40:28 -0800
Subject: [PATCH 79/94] Use null instead of NO_CREDENTIALS

---
 .../com/google/gcloud/AuthCredentials.java     | 18 +-----------------
 .../java/com/google/gcloud/ServiceOptions.java | 18 +++++++++++-------
 .../gcloud/datastore/SerializationTest.java    |  2 +-
 .../gcloud/storage/SerializationTest.java      |  4 +---
 .../google/gcloud/storage/StorageImplTest.java |  4 +---
 5 files changed, 15 insertions(+), 31 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
index 779a60fb8a1b..9aaee77ea243 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -136,8 +136,6 @@ public static class ServiceAccountAuthCredentials extends AuthCredentials {
     private final String account;
     private final PrivateKey privateKey;
 
-    private static final AuthCredentials NO_CREDENTIALS = new ServiceAccountAuthCredentials();
-
     private static class ServiceAccountAuthCredentialsState
         implements RestorableState, Serializable {
 
@@ -153,9 +151,6 @@ private ServiceAccountAuthCredentialsState(String account, PrivateKey privateKey
 
       @Override
       public AuthCredentials restore() {
-        if (account == null && privateKey == null) {
-          return NO_CREDENTIALS;
-        }
         return new ServiceAccountAuthCredentials(account, privateKey);
       }
 
@@ -180,16 +175,9 @@ public boolean equals(Object obj) {
       this.privateKey = checkNotNull(privateKey);
     }
 
-    ServiceAccountAuthCredentials() {
-      account = null;
-      privateKey = null;
-    }
-
     @Override
     protected GoogleCredentials credentials() {
-      return (privateKey == null)
-          ? new GoogleCredentials(null)
-          : new ServiceAccountCredentials(null, account, privateKey, null, null);
+      return new ServiceAccountCredentials(null, account, privateKey, null, null);
     }
 
     public String account() {
@@ -327,8 +315,4 @@ public static ServiceAccountAuthCredentials createForJson(InputStream jsonCreden
           "The given JSON Credentials Stream is not a service account credential.");
     }
   }
-
-  public static AuthCredentials noCredentials() {
-    return ServiceAccountAuthCredentials.NO_CREDENTIALS;
-  }
 }
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
index bc0e91d0a694..715b0b92ca98 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -312,8 +312,9 @@ protected ServiceOptions(Class> ser
     httpTransportFactory = firstNonNull(builder.httpTransportFactory,
         getFromServiceLoader(HttpTransportFactory.class, DefaultHttpTransportFactory.INSTANCE));
     httpTransportFactoryClassName = httpTransportFactory.getClass().getName();
-    authCredentials = firstNonNull(builder.authCredentials, defaultAuthCredentials());
-    authCredentialsState = authCredentials.capture();
+    authCredentials =
+        builder.authCredentials != null ? builder.authCredentials : defaultAuthCredentials();
+    authCredentialsState = authCredentials != null ? authCredentials.capture() : null;
     retryParams = builder.retryParams;
     serviceFactory = firstNonNull(builder.serviceFactory,
         getFromServiceLoader(serviceFactoryClass, defaultServiceFactory()));
@@ -349,7 +350,7 @@ private static AuthCredentials defaultAuthCredentials() {
     try {
       return AuthCredentials.createApplicationDefaults();
     } catch (Exception ex) {
-      return AuthCredentials.noCredentials();
+      return null;
     }
   }
 
@@ -509,12 +510,15 @@ public RetryParams retryParams() {
    * options.
    */
   public HttpRequestInitializer httpRequestInitializer() {
-    final HttpRequestInitializer baseRequestInitializer =
-        new HttpCredentialsAdapter(authCredentials().credentials().createScoped(scopes()));
+    final HttpRequestInitializer delegate = authCredentials() != null
+        ? new HttpCredentialsAdapter(authCredentials().credentials().createScoped(scopes()))
+        : null;
     return new HttpRequestInitializer() {
       @Override
       public void initialize(HttpRequest httpRequest) throws IOException {
-        baseRequestInitializer.initialize(httpRequest);
+        if (delegate != null) {
+          delegate.initialize(httpRequest);
+        }
         if (connectTimeout >= 0) {
           httpRequest.setConnectTimeout(connectTimeout);
         }
@@ -580,7 +584,7 @@ private void readObject(ObjectInputStream input) throws IOException, ClassNotFou
     httpTransportFactory = newInstance(httpTransportFactoryClassName);
     serviceFactory = newInstance(serviceFactoryClassName);
     serviceRpcFactory = newInstance(serviceRpcFactoryClassName);
-    authCredentials = authCredentialsState.restore();
+    authCredentials = authCredentialsState != null ? authCredentialsState.restore() : null;
   }
 
   private static  T newInstance(String className) throws IOException, ClassNotFoundException {
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 32e14fb47ea0..3976be2cc383 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -144,7 +144,7 @@ public void testServiceOptions() throws Exception {
     options = options.toBuilder()
         .namespace("ns1")
         .retryParams(RetryParams.defaultInstance())
-        .authCredentials(AuthCredentials.noCredentials())
+        .authCredentials(null)
         .force(true)
         .build();
     serializedCopy = serializeAndDeserialize(options);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
index 2d80191aeb2d..256f5fc23f9a 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -83,7 +83,7 @@ public void testServiceOptions() throws Exception {
     options = options.toBuilder()
         .projectId("p2")
         .retryParams(RetryParams.defaultInstance())
-        .authCredentials(AuthCredentials.noCredentials())
+        .authCredentials(null)
         .pathDelimiter(":")
         .build();
     serializedCopy = serializeAndDeserialize(options);
@@ -111,7 +111,6 @@ public void testReadChannelState() throws IOException, ClassNotFoundException {
     StorageOptions options = StorageOptions.builder()
         .projectId("p2")
         .retryParams(RetryParams.defaultInstance())
-        .authCredentials(AuthCredentials.noCredentials())
         .build();
     BlobReadChannel reader =
         new BlobReadChannelImpl(options, BlobId.of("b", "n"), EMPTY_RPC_OPTIONS);
@@ -127,7 +126,6 @@ public void testWriteChannelState() throws IOException, ClassNotFoundException {
     StorageOptions options = StorageOptions.builder()
         .projectId("p2")
         .retryParams(RetryParams.defaultInstance())
-        .authCredentials(AuthCredentials.noCredentials())
         .build();
     BlobWriteChannelImpl writer = new BlobWriteChannelImpl(
         options, BlobInfo.builder(BlobId.of("b", "n")).build(), "upload-id");
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
index 32a466a9d551..42671d37c60c 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
@@ -31,11 +31,10 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.io.BaseEncoding;
-import com.google.gcloud.AuthCredentials;
 import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
+import com.google.gcloud.Page;
 import com.google.gcloud.RetryParams;
 import com.google.gcloud.ServiceOptions;
-import com.google.gcloud.Page;
 import com.google.gcloud.spi.StorageRpc;
 import com.google.gcloud.spi.StorageRpc.Tuple;
 import com.google.gcloud.spi.StorageRpcFactory;
@@ -260,7 +259,6 @@ public void setUp() throws IOException, InterruptedException {
     EasyMock.replay(rpcFactoryMock);
     options = StorageOptions.builder()
         .projectId("projectId")
-        .authCredentials(AuthCredentials.noCredentials())
         .clock(TIME_SOURCE)
         .serviceRpcFactory(rpcFactoryMock)
         .build();

From 12078e04dabf1790f02ea17888bde24a7515d8bf Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Tue, 1 Dec 2015 14:07:31 -0800
Subject: [PATCH 80/94] Make AppEngineAuthCredentials Restorable

---
 .../com/google/gcloud/AuthCredentials.java    | 49 +++++++++----------
 .../google/gcloud/storage/StorageImpl.java    | 25 +++++-----
 2 files changed, 33 insertions(+), 41 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
index 9aaee77ea243..7f5d7a8ab89b 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -43,6 +43,8 @@ private static class AppEngineAuthCredentials extends AuthCredentials {
     private static class AppEngineCredentials extends GoogleCredentials {
 
       private final Object appIdentityService;
+      private final Method getAccessToken;
+      private final Method getAccessTokenResult;
       private final Collection scopes;
       private final boolean scopesRequired;
 
@@ -52,6 +54,12 @@ private static class AppEngineCredentials extends GoogleCredentials {
               Class.forName("com.google.appengine.api.appidentity.AppIdentityServiceFactory");
           Method method = factoryClass.getMethod("getAppIdentityService");
           this.appIdentityService = method.invoke(null);
+          Class serviceClass =
+              Class.forName("com.google.appengine.api.appidentity.AppIdentityService");
+          Class tokenResultClass = Class.forName(
+              "com.google.appengine.api.appidentity.AppIdentityService$GetAccessTokenResult");
+          this.getAccessTokenResult = serviceClass.getMethod("getAccessToken", Iterable.class);
+          this.getAccessToken = tokenResultClass.getMethod("getAccessToken");
           this.scopes = null;
           this.scopesRequired = true;
         } catch (Exception e) {
@@ -59,10 +67,13 @@ private static class AppEngineCredentials extends GoogleCredentials {
         }
       }
 
-      AppEngineCredentials(Collection scopes, Object appIdentityService) {
-          this.appIdentityService = appIdentityService;
-          this.scopes = scopes;
-          this.scopesRequired = (scopes == null || scopes.isEmpty());
+      AppEngineCredentials(Collection scopes, Object appIdentityService,
+          Method getAccessToken, Method getAccessTokenResult) {
+        this.appIdentityService = appIdentityService;
+        this.getAccessToken = getAccessToken;
+        this.getAccessTokenResult = getAccessTokenResult;
+        this.scopes = scopes;
+        this.scopesRequired = (scopes == null || scopes.isEmpty());
       }
 
       /**
@@ -74,13 +85,7 @@ public AccessToken refreshAccessToken() throws IOException {
           throw new IOException("AppEngineCredentials requires createScoped call before use.");
         }
         try {
-          Class serviceClass =
-              Class.forName("com.google.appengine.api.appidentity.AppIdentityService");
-          Class tokenResultClass = Class.forName(
-              "com.google.appengine.api.appidentity.AppIdentityService$GetAccessTokenResult");
-          Method getAccessTokenResult = serviceClass.getMethod("getAccessToken", Iterable.class);
           Object accessTokenResult = getAccessTokenResult.invoke(appIdentityService, scopes);
-          Method getAccessToken = tokenResultClass.getMethod("getAccessToken");
           String accessToken = (String) getAccessToken.invoke(accessTokenResult);
           return new AccessToken(accessToken, null);
         } catch (Exception e) {
@@ -95,7 +100,8 @@ public boolean createScopedRequired() {
 
       @Override
       public GoogleCredentials createScoped(Collection scopes) {
-        return new AppEngineCredentials(scopes, appIdentityService);
+        return new AppEngineCredentials(
+            scopes, appIdentityService, getAccessToken, getAccessTokenResult);
       }
     }
 
@@ -121,7 +127,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    protected GoogleCredentials credentials() {
+    public GoogleCredentials credentials() {
       return new AppEngineCredentials();
     }
 
@@ -176,7 +182,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    protected GoogleCredentials credentials() {
+    public GoogleCredentials credentials() {
       return new ServiceAccountCredentials(null, account, privateKey, null, null);
     }
 
@@ -232,26 +238,17 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    protected GoogleCredentials credentials() {
+    public GoogleCredentials credentials() {
       return googleCredentials;
     }
 
-    public ServiceAccountAuthCredentials toServiceAccountCredentials() {
-      if (googleCredentials instanceof ServiceAccountCredentials) {
-        ServiceAccountCredentials credentials = (ServiceAccountCredentials) googleCredentials;
-        return new ServiceAccountAuthCredentials(credentials.getClientEmail(),
-            credentials.getPrivateKey());
-      }
-      return null;
-    }
-
     @Override
     public RestorableState capture() {
       return STATE;
     }
   }
 
-  protected abstract GoogleCredentials credentials();
+  public abstract GoogleCredentials credentials();
 
   public static AuthCredentials createForAppEngine() {
     return AppEngineAuthCredentials.INSTANCE;
@@ -310,9 +307,7 @@ public static ServiceAccountAuthCredentials createForJson(InputStream jsonCreden
       return new ServiceAccountAuthCredentials(
           tempServiceAccountCredentials.getClientEmail(),
           tempServiceAccountCredentials.getPrivateKey());
-    } else {
-      throw new IOException(
-          "The given JSON Credentials Stream is not a service account credential.");
     }
+    throw new IOException("The given JSON Credentials Stream is not a service account credential.");
   }
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index 91a408657847..fdb643c725ac 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -31,6 +31,8 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.services.storage.model.StorageObject;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.auth.oauth2.ServiceAccountCredentials;
 import com.google.common.base.Function;
 import com.google.common.base.Functions;
 import com.google.common.collect.ImmutableList;
@@ -38,19 +40,17 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
 import com.google.common.hash.Hashing;
 import com.google.common.io.BaseEncoding;
 import com.google.common.primitives.Ints;
 import com.google.gcloud.AuthCredentials;
-import com.google.gcloud.AuthCredentials.ApplicationDefaultAuthCredentials;
 import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
-import com.google.gcloud.PageImpl;
 import com.google.gcloud.BaseService;
 import com.google.gcloud.ExceptionHandler;
 import com.google.gcloud.ExceptionHandler.Interceptor;
-import com.google.gcloud.RetryHelper.RetryHelperException;
 import com.google.gcloud.Page;
+import com.google.gcloud.PageImpl;
+import com.google.gcloud.RetryHelper.RetryHelperException;
 import com.google.gcloud.spi.StorageRpc;
 import com.google.gcloud.spi.StorageRpc.RewriteResponse;
 import com.google.gcloud.spi.StorageRpc.Tuple;
@@ -71,7 +71,6 @@
 import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 
@@ -566,15 +565,13 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio
     ServiceAccountAuthCredentials cred =
         (ServiceAccountAuthCredentials) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED);
     if (cred == null) {
-      AuthCredentials serviceCred = this.options().authCredentials();
-      if (serviceCred instanceof ServiceAccountAuthCredentials) {
-        cred = (ServiceAccountAuthCredentials) serviceCred;
-      } else {
-        if (serviceCred instanceof ApplicationDefaultAuthCredentials) {
-          cred = ((ApplicationDefaultAuthCredentials) serviceCred).toServiceAccountCredentials();
-        }
-      }
-      checkArgument(cred != null, "Signing key was not provided and could not be derived");
+      AuthCredentials authCredentials = this.options().authCredentials();
+      GoogleCredentials serviceCred =
+          authCredentials != null ? authCredentials.credentials() : null;
+      checkArgument(
+          serviceCred instanceof ServiceAccountCredentials,
+          "Signing key was not provided and could not be derived");
+      cred = (ServiceAccountAuthCredentials) authCredentials;
     }
     // construct signature - see https://cloud.google.com/storage/docs/access-control#Signed-URLs
     StringBuilder stBuilder = new StringBuilder();

From af94b5312daa978b693ce38b5f2314a335f3d531 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Tue, 1 Dec 2015 15:33:03 -0800
Subject: [PATCH 81/94] cleanup + cast fix

---
 .../com/google/gcloud/AuthCredentials.java    | 24 ++++++++-----------
 .../google/gcloud/storage/StorageImpl.java    | 16 +++++++------
 2 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
index 7f5d7a8ab89b..f3a0024129d7 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -46,7 +46,6 @@ private static class AppEngineCredentials extends GoogleCredentials {
       private final Method getAccessToken;
       private final Method getAccessTokenResult;
       private final Collection scopes;
-      private final boolean scopesRequired;
 
       AppEngineCredentials() {
         try {
@@ -61,19 +60,16 @@ private static class AppEngineCredentials extends GoogleCredentials {
           this.getAccessTokenResult = serviceClass.getMethod("getAccessToken", Iterable.class);
           this.getAccessToken = tokenResultClass.getMethod("getAccessToken");
           this.scopes = null;
-          this.scopesRequired = true;
         } catch (Exception e) {
-          throw new RuntimeException("Could not create AppEngineCredentials using reflection.");
+          throw new RuntimeException("Could not create AppEngineCredentials.", e);
         }
       }
 
-      AppEngineCredentials(Collection scopes, Object appIdentityService,
-          Method getAccessToken, Method getAccessTokenResult) {
-        this.appIdentityService = appIdentityService;
-        this.getAccessToken = getAccessToken;
-        this.getAccessTokenResult = getAccessTokenResult;
+      AppEngineCredentials(Collection scopes, AppEngineCredentials unscoped) {
+        this.appIdentityService = unscoped.appIdentityService;
+        this.getAccessToken = unscoped.getAccessToken;
+        this.getAccessTokenResult = unscoped.getAccessTokenResult;
         this.scopes = scopes;
-        this.scopesRequired = (scopes == null || scopes.isEmpty());
       }
 
       /**
@@ -89,19 +85,18 @@ public AccessToken refreshAccessToken() throws IOException {
           String accessToken = (String) getAccessToken.invoke(accessTokenResult);
           return new AccessToken(accessToken, null);
         } catch (Exception e) {
-          throw new RuntimeException("Could not get the access token using reflection.");
+          throw new IOException("Could not get the access token.", e);
         }
       }
 
       @Override
       public boolean createScopedRequired() {
-        return scopesRequired;
+        return scopes == null || scopes.isEmpty();
       }
 
       @Override
       public GoogleCredentials createScoped(Collection scopes) {
-        return new AppEngineCredentials(
-            scopes, appIdentityService, getAccessToken, getAccessTokenResult);
+        return new AppEngineCredentials(scopes, this);
       }
     }
 
@@ -308,6 +303,7 @@ public static ServiceAccountAuthCredentials createForJson(InputStream jsonCreden
           tempServiceAccountCredentials.getClientEmail(),
           tempServiceAccountCredentials.getPrivateKey());
     }
-    throw new IOException("The given JSON Credentials Stream is not a service account credential.");
+    throw new IOException(
+        "The given JSON Credentials Stream is not for a service account credential.");
   }
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index fdb643c725ac..9207e4905c54 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -562,16 +562,18 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio
     for (SignUrlOption option : options) {
       optionMap.put(option.option(), option.value());
     }
-    ServiceAccountAuthCredentials cred =
+    ServiceAccountAuthCredentials serviceAccountAuthCred =
         (ServiceAccountAuthCredentials) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED);
-    if (cred == null) {
-      AuthCredentials authCredentials = this.options().authCredentials();
+    ServiceAccountCredentials cred = (ServiceAccountCredentials) (serviceAccountAuthCred != null
+        ? serviceAccountAuthCred.credentials() : null);
+    if (serviceAccountAuthCred == null) {
+      AuthCredentials authCred = this.options().authCredentials();
       GoogleCredentials serviceCred =
-          authCredentials != null ? authCredentials.credentials() : null;
+          authCred != null ? authCred.credentials() : null;
       checkArgument(
           serviceCred instanceof ServiceAccountCredentials,
           "Signing key was not provided and could not be derived");
-      cred = (ServiceAccountAuthCredentials) authCredentials;
+      cred = (ServiceAccountCredentials) serviceCred;
     }
     // construct signature - see https://cloud.google.com/storage/docs/access-control#Signed-URLs
     StringBuilder stBuilder = new StringBuilder();
@@ -607,12 +609,12 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio
     stBuilder.append(path);
     try {
       Signature signer = Signature.getInstance("SHA256withRSA");
-      signer.initSign(cred.privateKey());
+      signer.initSign(cred.getPrivateKey());
       signer.update(stBuilder.toString().getBytes(UTF_8));
       String signature =
           URLEncoder.encode(BaseEncoding.base64().encode(signer.sign()), UTF_8.name());
       stBuilder = new StringBuilder("https://storage.googleapis.com").append(path);
-      stBuilder.append("?GoogleAccessId=").append(cred.account());
+      stBuilder.append("?GoogleAccessId=").append(cred.getClientEmail());
       stBuilder.append("&Expires=").append(expiration);
       stBuilder.append("&Signature=").append(signature);
       return new URL(stBuilder.toString());

From 18cbb774258a77172a988d7a84c5e559d57b0c95 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Tue, 1 Dec 2015 15:54:32 -0800
Subject: [PATCH 82/94] style fix

---
 .../com/google/gcloud/storage/StorageImpl.java | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index 9207e4905c54..7ff8ccd4d68d 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -31,7 +31,6 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.services.storage.model.StorageObject;
-import com.google.auth.oauth2.GoogleCredentials;
 import com.google.auth.oauth2.ServiceAccountCredentials;
 import com.google.common.base.Function;
 import com.google.common.base.Functions;
@@ -43,7 +42,6 @@
 import com.google.common.hash.Hashing;
 import com.google.common.io.BaseEncoding;
 import com.google.common.primitives.Ints;
-import com.google.gcloud.AuthCredentials;
 import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
 import com.google.gcloud.BaseService;
 import com.google.gcloud.ExceptionHandler;
@@ -562,18 +560,16 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio
     for (SignUrlOption option : options) {
       optionMap.put(option.option(), option.value());
     }
-    ServiceAccountAuthCredentials serviceAccountAuthCred =
+    ServiceAccountAuthCredentials authCred =
         (ServiceAccountAuthCredentials) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED);
-    ServiceAccountCredentials cred = (ServiceAccountCredentials) (serviceAccountAuthCred != null
-        ? serviceAccountAuthCred.credentials() : null);
-    if (serviceAccountAuthCred == null) {
-      AuthCredentials authCred = this.options().authCredentials();
-      GoogleCredentials serviceCred =
-          authCred != null ? authCred.credentials() : null;
+    ServiceAccountCredentials cred =
+        (ServiceAccountCredentials) (authCred != null ? authCred.credentials() : null);
+    if (authCred == null) {
       checkArgument(
-          serviceCred instanceof ServiceAccountCredentials,
+          this.options().authCredentials() != null
+          && this.options().authCredentials().credentials() instanceof ServiceAccountCredentials,
           "Signing key was not provided and could not be derived");
-      cred = (ServiceAccountCredentials) serviceCred;
+      cred = (ServiceAccountCredentials) this.options().authCredentials().credentials();
     }
     // construct signature - see https://cloud.google.com/storage/docs/access-control#Signed-URLs
     StringBuilder stBuilder = new StringBuilder();

From 942ceb40e3281593afb8da78ced1a9e476284c9c Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Tue, 1 Dec 2015 16:04:32 -0800
Subject: [PATCH 83/94] more style fixes

---
 .../src/main/java/com/google/gcloud/AuthCredentials.java       | 2 +-
 .../src/main/java/com/google/gcloud/storage/StorageImpl.java   | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
index f3a0024129d7..dab6b928374a 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -177,7 +177,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public GoogleCredentials credentials() {
+    public ServiceAccountCredentials credentials() {
       return new ServiceAccountCredentials(null, account, privateKey, null, null);
     }
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index 7ff8ccd4d68d..e31c4073548e 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -562,8 +562,7 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio
     }
     ServiceAccountAuthCredentials authCred =
         (ServiceAccountAuthCredentials) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED);
-    ServiceAccountCredentials cred =
-        (ServiceAccountCredentials) (authCred != null ? authCred.credentials() : null);
+    ServiceAccountCredentials cred = authCred != null ? authCred.credentials() : null;
     if (authCred == null) {
       checkArgument(
           this.options().authCredentials() != null

From 3980ba7167961e92c954744f4de6822eeae5b5f6 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Tue, 1 Dec 2015 17:07:46 -0800
Subject: [PATCH 84/94] update version to 0.1.0

---
 gcloud-java-core/pom.xml      | 2 +-
 gcloud-java-datastore/pom.xml | 2 +-
 gcloud-java-examples/pom.xml  | 2 +-
 gcloud-java-storage/pom.xml   | 2 +-
 gcloud-java/pom.xml           | 2 +-
 pom.xml                       | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml
index c0248004d335..4fcb0b27659a 100644
--- a/gcloud-java-core/pom.xml
+++ b/gcloud-java-core/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.0.13-SNAPSHOT
+    0.1.0
   
   
     gcloud-java-core
diff --git a/gcloud-java-datastore/pom.xml b/gcloud-java-datastore/pom.xml
index 4bf5caab2ff8..426978b67404 100644
--- a/gcloud-java-datastore/pom.xml
+++ b/gcloud-java-datastore/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.0.13-SNAPSHOT
+    0.1.0
   
   
     gcloud-java-datastore
diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml
index 2b02be47cc40..d3ae0fadc731 100644
--- a/gcloud-java-examples/pom.xml
+++ b/gcloud-java-examples/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.0.13-SNAPSHOT
+    0.1.0
   
   
     gcloud-java-examples
diff --git a/gcloud-java-storage/pom.xml b/gcloud-java-storage/pom.xml
index 3e0b6a9490a8..1521b0846684 100644
--- a/gcloud-java-storage/pom.xml
+++ b/gcloud-java-storage/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.0.13-SNAPSHOT
+    0.1.0
   
   
     gcloud-java-storage
diff --git a/gcloud-java/pom.xml b/gcloud-java/pom.xml
index 214094dcd104..a44297fd3b7d 100644
--- a/gcloud-java/pom.xml
+++ b/gcloud-java/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.0.13-SNAPSHOT
+    0.1.0
   
   
     
diff --git a/pom.xml b/pom.xml
index 234290a6da44..ba33bc5ce5de 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   com.google.gcloud
   gcloud-java-pom
   pom
-  0.0.13-SNAPSHOT
+  0.1.0
   GCloud Java
   https://github.com/GoogleCloudPlatform/gcloud-java
   

From a615317f7424ed58621b1f65d5c4d8cbbe8a6ed8 Mon Sep 17 00:00:00 2001
From: travis-ci 
Date: Wed, 2 Dec 2015 01:19:11 +0000
Subject: [PATCH 85/94] Updating version in README files. [ci skip]

---
 README.md                       | 6 +++---
 gcloud-java-core/README.md      | 6 +++---
 gcloud-java-datastore/README.md | 6 +++---
 gcloud-java-examples/README.md  | 6 +++---
 gcloud-java-storage/README.md   | 6 +++---
 gcloud-java/README.md           | 6 +++---
 6 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index f6104e64a9d3..123e4cf76bb3 100644
--- a/README.md
+++ b/README.md
@@ -25,16 +25,16 @@ If you are using Maven, add this to your pom.xml file
 
   com.google.gcloud
   gcloud-java
-  0.0.12
+  0.1.0
 
 ```
 If you are using Gradle, add this to your dependencies
 ```Groovy
-compile 'com.google.gcloud:gcloud-java:jar:0.0.12'
+compile 'com.google.gcloud:gcloud-java:jar:0.1.0'
 ```
 If you are using SBT, add this to your dependencies
 ```Scala
-libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.12"
+libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.1.0"
 ```
 
 Example Applications
diff --git a/gcloud-java-core/README.md b/gcloud-java-core/README.md
index 96b7a4e82021..3aea0f16efc5 100644
--- a/gcloud-java-core/README.md
+++ b/gcloud-java-core/README.md
@@ -17,16 +17,16 @@ If you are using Maven, add this to your pom.xml file
 
   com.google.gcloud
   gcloud-java-core
-  0.0.12
+  0.1.0
 
 ```
 If you are using Gradle, add this to your dependencies
 ```Groovy
-compile 'com.google.gcloud:gcloud-java-core:jar:0.0.12'
+compile 'com.google.gcloud:gcloud-java-core:jar:0.1.0'
 ```
 If you are using SBT, add this to your dependencies
 ```Scala
-libraryDependencies += "com.google.gcloud" % "gcloud-java-core" % "0.0.12"
+libraryDependencies += "com.google.gcloud" % "gcloud-java-core" % "0.1.0"
 ```
 
 Troubleshooting
diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md
index aeaae2b621c6..9777668f8d0e 100644
--- a/gcloud-java-datastore/README.md
+++ b/gcloud-java-datastore/README.md
@@ -20,16 +20,16 @@ If you are using Maven, add this to your pom.xml file
 
   com.google.gcloud
   gcloud-java-datastore
-  0.0.12
+  0.1.0
 
 ```
 If you are using Gradle, add this to your dependencies
 ```Groovy
-compile 'com.google.gcloud:gcloud-java-datastore:jar:0.0.12'
+compile 'com.google.gcloud:gcloud-java-datastore:jar:0.1.0'
 ```
 If you are using SBT, add this to your dependencies
 ```Scala
-libraryDependencies += "com.google.gcloud" % "gcloud-java-datastore" % "0.0.12"
+libraryDependencies += "com.google.gcloud" % "gcloud-java-datastore" % "0.1.0"
 ```
 
 Example Application
diff --git a/gcloud-java-examples/README.md b/gcloud-java-examples/README.md
index 2ccec3e81571..2a451db33036 100644
--- a/gcloud-java-examples/README.md
+++ b/gcloud-java-examples/README.md
@@ -17,16 +17,16 @@ If you are using Maven, add this to your pom.xml file
 
   com.google.gcloud
   gcloud-java-examples
-  0.0.12
+  0.1.0
 
 ```
 If you are using Gradle, add this to your dependencies
 ```Groovy
-compile 'com.google.gcloud:gcloud-java-examples:jar:0.0.12'
+compile 'com.google.gcloud:gcloud-java-examples:jar:0.1.0'
 ```
 If you are using SBT, add this to your dependencies
 ```Scala
-libraryDependencies += "com.google.gcloud" % "gcloud-java-examples" % "0.0.12"
+libraryDependencies += "com.google.gcloud" % "gcloud-java-examples" % "0.1.0"
 ```
 
 To run examples from your command line:
diff --git a/gcloud-java-storage/README.md b/gcloud-java-storage/README.md
index d679d3da9bc1..01d34f318a42 100644
--- a/gcloud-java-storage/README.md
+++ b/gcloud-java-storage/README.md
@@ -20,16 +20,16 @@ If you are using Maven, add this to your pom.xml file
 
   com.google.gcloud
   gcloud-java-storage
-  0.0.12
+  0.1.0
 
 ```
 If you are using Gradle, add this to your dependencies
 ```Groovy
-compile 'com.google.gcloud:gcloud-java-storage:jar:0.0.12'
+compile 'com.google.gcloud:gcloud-java-storage:jar:0.1.0'
 ```
 If you are using SBT, add this to your dependencies
 ```Scala
-libraryDependencies += "com.google.gcloud" % "gcloud-java-storage" % "0.0.12"
+libraryDependencies += "com.google.gcloud" % "gcloud-java-storage" % "0.1.0"
 ```
 
 Example Application
diff --git a/gcloud-java/README.md b/gcloud-java/README.md
index e381ca80cdaa..98cd7a8f6e85 100644
--- a/gcloud-java/README.md
+++ b/gcloud-java/README.md
@@ -25,16 +25,16 @@ If you are using Maven, add this to your pom.xml file
 
   com.google.gcloud
   gcloud-java
-  0.0.12
+  0.1.0
 
 ```
 If you are using Gradle, add this to your dependencies
 ```Groovy
-compile 'com.google.gcloud:gcloud-java:jar:0.0.12'
+compile 'com.google.gcloud:gcloud-java:jar:0.1.0'
 ```
 If you are using SBT, add this to your dependencies
 ```Scala
-libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.0.12"
+libraryDependencies += "com.google.gcloud" % "gcloud-java" % "0.1.0"
 ```
 
 Troubleshooting

From e1c770e95465cae6baec0c0f2be6f7734d0f77fe Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Tue, 1 Dec 2015 17:25:56 -0800
Subject: [PATCH 86/94] Update version to snapshot

---
 gcloud-java-core/pom.xml      | 2 +-
 gcloud-java-datastore/pom.xml | 2 +-
 gcloud-java-examples/pom.xml  | 2 +-
 gcloud-java-storage/pom.xml   | 2 +-
 gcloud-java/pom.xml           | 2 +-
 pom.xml                       | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml
index 4fcb0b27659a..83c8dbb00e71 100644
--- a/gcloud-java-core/pom.xml
+++ b/gcloud-java-core/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.1.0
+    0.1.1-SNAPSHOT
   
   
     gcloud-java-core
diff --git a/gcloud-java-datastore/pom.xml b/gcloud-java-datastore/pom.xml
index 426978b67404..163fd8accf7a 100644
--- a/gcloud-java-datastore/pom.xml
+++ b/gcloud-java-datastore/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.1.0
+    0.1.1-SNAPSHOT
   
   
     gcloud-java-datastore
diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml
index d3ae0fadc731..4f113dd78ec0 100644
--- a/gcloud-java-examples/pom.xml
+++ b/gcloud-java-examples/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.1.0
+    0.1.1-SNAPSHOT
   
   
     gcloud-java-examples
diff --git a/gcloud-java-storage/pom.xml b/gcloud-java-storage/pom.xml
index 1521b0846684..907ec66d7214 100644
--- a/gcloud-java-storage/pom.xml
+++ b/gcloud-java-storage/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.1.0
+    0.1.1-SNAPSHOT
   
   
     gcloud-java-storage
diff --git a/gcloud-java/pom.xml b/gcloud-java/pom.xml
index a44297fd3b7d..7d8e251b54fb 100644
--- a/gcloud-java/pom.xml
+++ b/gcloud-java/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.1.0
+    0.1.1-SNAPSHOT
   
   
     
diff --git a/pom.xml b/pom.xml
index ba33bc5ce5de..5b11a09fb382 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   com.google.gcloud
   gcloud-java-pom
   pom
-  0.1.0
+  0.1.1-SNAPSHOT
   GCloud Java
   https://github.com/GoogleCloudPlatform/gcloud-java
   

From 6c472d5d8b83575174e3efc83fb24c72e95c3694 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Wed, 2 Dec 2015 12:48:33 -0800
Subject: [PATCH 87/94] Add release notes to RELEASING

---
 RELEASING.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/RELEASING.md b/RELEASING.md
index 59b6559b9f95..dd319b7d0dc7 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -13,7 +13,11 @@ The PR should look something like [#225](https://github.com/GoogleCloudPlatform/
 3. Before moving on, verify that the artifacts have successfully been pushed to the Maven Central Repository.  Open Travis CI, click the ["Build History" tab](https://travis-ci.org/GoogleCloudPlatform/gcloud-java/builds), and open the second build's logs for Step 2's PR.  Be sure that you are not opening the "Pull Request" build logs.  When the build finishes, scroll to the end of the log and verify that the artifacts were successfully staged and deployed.  You can also search for `gcloud-java` on the [Sonatype website](https://oss.sonatype.org/#nexus-search;quick~gcloud-java) and check the latest version number.  If the deployment didn't succeed because of a flaky test, rerun the build.
 
 4. Create a release on Github manually.
-Go to the [releases page](https://github.com/GoogleCloudPlatform/gcloud-java/releases) and click "Draft a new release."  Use `vX.Y.Z` as the "Tag Version" and `X.Y.Z` as the "Release Title", where `X.Y.Z` is the release version as listed in the `pom.xml` files.
+Go to the [releases page](https://github.com/GoogleCloudPlatform/gcloud-java/releases) and click "Draft a new release."  Use `vX.Y.Z` as the "Tag Version" and `X.Y.Z` as the "Release Title", where `X.Y.Z` is the release version as listed in the `pom.xml` files.  In the description section, write brief notes on what has changed.  For an example, see the [0.1.0 release](https://github.com/GoogleCloudPlatform/gcloud-java/releases/tag/v0.1.0).  You can see what has changed by looking through the merged master branch pull requests or by using the `git log` command.  Here is an example of the log command to get non-merge commits between v0.0.12 and v0.1.0:
+
+  ```
+  git --no-pager log v0.0.12..v0.1.0 --pretty=oneline --abbrev-commit | grep -v " Merge pull request"
+  ```
 
 5. Run `utilities/update_pom_version.sh` again (to include "-SNAPSHOT" in the project version).
 As mentioned before, there is an optional version argument.  By default, the script will update the version from "X.Y.Z" to "X.Y.Z+1-SNAPSHOT".  Suppose a different version is desired, for example X+1.0.0-SNAPSHOT.  Then the appropriate command to run would be `utilities/update_pom_version.sh X+1.0.0-SNAPSHOT`.

From 231b84a6ab5ed55eb6cda9ee9a07b3cbacc99763 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Wed, 2 Dec 2015 14:48:35 -0800
Subject: [PATCH 88/94] Update release notes procedure

---
 RELEASING.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/RELEASING.md b/RELEASING.md
index dd319b7d0dc7..142dd377a642 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -12,13 +12,15 @@ The PR should look something like [#225](https://github.com/GoogleCloudPlatform/
 
 3. Before moving on, verify that the artifacts have successfully been pushed to the Maven Central Repository.  Open Travis CI, click the ["Build History" tab](https://travis-ci.org/GoogleCloudPlatform/gcloud-java/builds), and open the second build's logs for Step 2's PR.  Be sure that you are not opening the "Pull Request" build logs.  When the build finishes, scroll to the end of the log and verify that the artifacts were successfully staged and deployed.  You can also search for `gcloud-java` on the [Sonatype website](https://oss.sonatype.org/#nexus-search;quick~gcloud-java) and check the latest version number.  If the deployment didn't succeed because of a flaky test, rerun the build.
 
-4. Create a release on Github manually.
-Go to the [releases page](https://github.com/GoogleCloudPlatform/gcloud-java/releases) and click "Draft a new release."  Use `vX.Y.Z` as the "Tag Version" and `X.Y.Z` as the "Release Title", where `X.Y.Z` is the release version as listed in the `pom.xml` files.  In the description section, write brief notes on what has changed.  For an example, see the [0.1.0 release](https://github.com/GoogleCloudPlatform/gcloud-java/releases/tag/v0.1.0).  You can see what has changed by looking through the merged master branch pull requests or by using the `git log` command.  Here is an example of the log command to get non-merge commits between v0.0.12 and v0.1.0:
+4. Publish a release on Github manually.
+Go to the [releases page](https://github.com/GoogleCloudPlatform/gcloud-java/releases) and open the appropriate release draft. Make sure the "Tag Version" is `vX.Y.Z` and the "Release Title" is `X.Y.Z`, where `X.Y.Z` is the release version as listed in the `pom.xml` files.  The draft should already have all changes that impact users since the previous release.  To double check this, you can use the `git log` command and look through the merged master branch pull requests.  Here is an example of the log command to get non-merge commits between v0.0.12 and v0.1.0:
 
   ```
-  git --no-pager log v0.0.12..v0.1.0 --pretty=oneline --abbrev-commit | grep -v " Merge pull request"
+  git --no-pager log v0.0.12..v0.1.0 --pretty=oneline --abbrev-commit --no-merges
   ```
 
+  Ensure that the format is consistent with previous releases (for an example, see the [0.1.0 release](https://github.com/GoogleCloudPlatform/gcloud-java/releases/tag/v0.1.0)).  After adding any missing updates and reformatting as necessary, publish the "draft".
+
 5. Run `utilities/update_pom_version.sh` again (to include "-SNAPSHOT" in the project version).
 As mentioned before, there is an optional version argument.  By default, the script will update the version from "X.Y.Z" to "X.Y.Z+1-SNAPSHOT".  Suppose a different version is desired, for example X+1.0.0-SNAPSHOT.  Then the appropriate command to run would be `utilities/update_pom_version.sh X+1.0.0-SNAPSHOT`.
 

From abf6b0640e78128dffb5729e8803be372259db6d Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Wed, 2 Dec 2015 15:09:30 -0800
Subject: [PATCH 89/94] add note about creating new draft

---
 RELEASING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/RELEASING.md b/RELEASING.md
index 142dd377a642..586ebeb9b53b 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -19,7 +19,7 @@ Go to the [releases page](https://github.com/GoogleCloudPlatform/gcloud-java/rel
   git --no-pager log v0.0.12..v0.1.0 --pretty=oneline --abbrev-commit --no-merges
   ```
 
-  Ensure that the format is consistent with previous releases (for an example, see the [0.1.0 release](https://github.com/GoogleCloudPlatform/gcloud-java/releases/tag/v0.1.0)).  After adding any missing updates and reformatting as necessary, publish the "draft".
+  Ensure that the format is consistent with previous releases (for an example, see the [0.1.0 release](https://github.com/GoogleCloudPlatform/gcloud-java/releases/tag/v0.1.0)).  After adding any missing updates and reformatting as necessary, publish the draft.  Finally, create a new draft for the next release.
 
 5. Run `utilities/update_pom_version.sh` again (to include "-SNAPSHOT" in the project version).
 As mentioned before, there is an optional version argument.  By default, the script will update the version from "X.Y.Z" to "X.Y.Z+1-SNAPSHOT".  Suppose a different version is desired, for example X+1.0.0-SNAPSHOT.  Then the appropriate command to run would be `utilities/update_pom_version.sh X+1.0.0-SNAPSHOT`.

From ba9059c7cfdcc32256511dde2bc27a5e74f54a61 Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Thu, 3 Dec 2015 18:15:01 +0100
Subject: [PATCH 90/94] Fix gradle dependency in README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 123e4cf76bb3..0fb8d82623b5 100644
--- a/README.md
+++ b/README.md
@@ -30,7 +30,7 @@ If you are using Maven, add this to your pom.xml file
 ```
 If you are using Gradle, add this to your dependencies
 ```Groovy
-compile 'com.google.gcloud:gcloud-java:jar:0.1.0'
+compile 'com.google.gcloud:gcloud-java:0.1.0'
 ```
 If you are using SBT, add this to your dependencies
 ```Scala

From c36b857f7eabee5d069899195b613271a9c31332 Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Wed, 9 Dec 2015 17:54:19 -0800
Subject: [PATCH 91/94] fix checkstyle issues and warnings

---
 .../src/main/java/com/google/gcloud/PageImpl.java   |  1 +
 .../main/java/com/google/gcloud/ServiceFactory.java |  1 +
 .../main/java/com/google/gcloud/ServiceOptions.java |  8 ++++++--
 .../com/google/gcloud/spi/ServiceRpcFactory.java    |  1 +
 .../test/java/com/google/gcloud/PageImplTest.java   |  2 --
 .../google/gcloud/datastore/DatastoreException.java |  3 ++-
 .../google/gcloud/datastore/DatastoreOptions.java   |  3 +++
 .../gcloud/datastore/DatastoreOptionsTest.java      |  4 +---
 .../com/google/gcloud/datastore/DatastoreTest.java  |  2 +-
 .../java/com/google/gcloud/datastore/ValueTest.java |  3 +++
 .../com/google/gcloud/examples/StorageExample.java  |  2 +-
 .../com/google/gcloud/spi/DefaultStorageRpc.java    |  6 ++++--
 .../main/java/com/google/gcloud/storage/Blob.java   |  2 +-
 .../google/gcloud/storage/BlobWriteChannelImpl.java |  1 -
 .../main/java/com/google/gcloud/storage/Bucket.java |  2 +-
 .../java/com/google/gcloud/storage/BucketInfo.java  |  1 +
 .../java/com/google/gcloud/storage/Storage.java     |  3 +--
 .../java/com/google/gcloud/storage/StorageImpl.java |  4 ++--
 .../com/google/gcloud/storage/StorageOptions.java   |  3 +++
 .../gcloud/storage/BlobReadChannelImplTest.java     | 13 +++++++------
 .../java/com/google/gcloud/storage/BlobTest.java    |  2 +-
 .../gcloud/storage/BlobWriteChannelImplTest.java    |  3 ++-
 .../com/google/gcloud/storage/BucketInfoTest.java   |  1 -
 .../java/com/google/gcloud/storage/BucketTest.java  |  6 +++---
 .../com/google/gcloud/storage/CopyWriterTest.java   |  9 ++++-----
 .../com/google/gcloud/storage/ITStorageTest.java    | 10 ++++------
 .../google/gcloud/storage/SerializationTest.java    |  2 ++
 .../com/google/gcloud/storage/StorageImplTest.java  |  4 ++--
 28 files changed, 58 insertions(+), 44 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java b/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java
index 5e83b53b33a9..97c2931e0cfb 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java
@@ -71,6 +71,7 @@ public PageImpl(NextPageFetcher pageFetcher, String cursor, Iterable resul
     this.results = results;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public Iterable values() {
     return results == null ? Collections.EMPTY_LIST : results;
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceFactory.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceFactory.java
index b59fc1e9a10e..3828bf30260b 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceFactory.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceFactory.java
@@ -22,6 +22,7 @@
  * Implementation must provide a public no-arg constructor.
  * Loading of a factory implementation is done via {@link java.util.ServiceLoader}.
  */
+@SuppressWarnings("rawtypes")
 public interface ServiceFactory {
 
   ServiceT create(ServiceOptionsT serviceOptions);
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
index 715b0b92ca98..9e5d6730f551 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -53,6 +53,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+@SuppressWarnings("rawtypes")
 public abstract class ServiceOptions<
     ServiceT extends Service,
     ServiceRpcT,
@@ -305,8 +306,8 @@ protected ServiceOptions(Class> ser
     if (projectIdRequired()) {
       checkArgument(
           projectId != null,
-          "A project ID is required for this service but could not be determined from the builder or "
-          + "the environment.  Please set a project ID using the builder.");
+          "A project ID is required for this service but could not be determined from the builder "
+          + "or the environment.  Please set a project ID using the builder.");
     }
     host = firstNonNull(builder.host, defaultHost());
     httpTransportFactory = firstNonNull(builder.httpTransportFactory,
@@ -453,6 +454,7 @@ protected static String getAppEngineProjectId() {
     }
   }
 
+  @SuppressWarnings("unchecked")
   public ServiceT service() {
     if (service == null) {
       service = serviceFactory.create((OptionsT) this);
@@ -460,6 +462,7 @@ public ServiceT service() {
     return service;
   }
 
+  @SuppressWarnings("unchecked")
   public ServiceRpcT rpc() {
     if (rpc == null) {
       rpc = serviceRpcFactory.create((OptionsT) this);
@@ -587,6 +590,7 @@ private void readObject(ObjectInputStream input) throws IOException, ClassNotFou
     authCredentials = authCredentialsState != null ? authCredentialsState.restore() : null;
   }
 
+  @SuppressWarnings("unchecked")
   private static  T newInstance(String className) throws IOException, ClassNotFoundException {
     try {
       return (T) Class.forName(className).newInstance();
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
index d20b690167a1..d19f6047e4b2 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
@@ -24,6 +24,7 @@
  * Implementation must provide a public no-arg constructor.
  * Loading of a factory implementation is done via {@link java.util.ServiceLoader}.
  */
+@SuppressWarnings("rawtypes")
 public interface ServiceRpcFactory {
 
   ServiceRpcT create(OptionsT options);
diff --git a/gcloud-java-core/src/test/java/com/google/gcloud/PageImplTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/PageImplTest.java
index fb289186de8d..4389171fb49c 100644
--- a/gcloud-java-core/src/test/java/com/google/gcloud/PageImplTest.java
+++ b/gcloud-java-core/src/test/java/com/google/gcloud/PageImplTest.java
@@ -22,8 +22,6 @@
 
 import org.junit.Test;
 
-import java.util.Collections;
-
 public class PageImplTest {
 
   private static final ImmutableList VALUES = ImmutableList.of("1", "2");
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java
index dded1d11875e..ebef6b44f6b6 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java
@@ -146,7 +146,8 @@ static DatastoreException translateAndThrow(DatastoreRpcException exception) {
    * @throws DatastoreException every time
    */
   static DatastoreException throwInvalidRequest(String massage, Object... params) {
-    throw new DatastoreException(DatastoreError.FAILED_PRECONDITION, String.format(massage, params));
+    throw new DatastoreException(
+        DatastoreError.FAILED_PRECONDITION, String.format(massage, params));
   }
 
   static DatastoreException propagateUserException(Exception ex) {
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java
index ee15393a8048..227419d8acc8 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java
@@ -157,11 +157,13 @@ protected String defaultProject() {
     return projectId != null ? projectId : super.defaultProject();
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   protected DatastoreFactory defaultServiceFactory() {
     return DefaultDatastoreFactory.INSTANCE;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   protected DatastoreRpcFactory defaultRpcFactory() {
     return DefaultDatastoreRpcFactory.INSTANCE;
@@ -199,6 +201,7 @@ protected Set scopes() {
     return SCOPES;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public Builder toBuilder() {
     return new Builder(this);
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java
index 78536e3f45cb..284a9d322793 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java
@@ -30,8 +30,6 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.IOException;
-
 public class DatastoreOptionsTest {
 
   private static final String PROJECT_ID = "project_id";
@@ -41,7 +39,7 @@ public class DatastoreOptionsTest {
   private DatastoreOptions.Builder options;
 
   @Before
-  public void setUp() throws IOException, InterruptedException {
+  public void setUp() {
     datastoreRpcFactory = EasyMock.createMock(DatastoreRpcFactory.class);
     datastoreRpc = EasyMock.createMock(DatastoreRpc.class);
     options = DatastoreOptions.builder()
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java
index 162104c0dd84..029b161c1c38 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java
@@ -121,7 +121,7 @@ public static void beforeClass() throws IOException, InterruptedException {
   }
 
   @Before
-  public void setUp() throws IOException, InterruptedException {
+  public void setUp() {
     options = DatastoreOptions.builder()
         .projectId(PROJECT_ID)
         .host("http://localhost:" + PORT)
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
index 199b3f90f442..3867e194d060 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
@@ -60,11 +60,13 @@ public class ValueTest {
 
   private ImmutableMap> typeToValue;
 
+  @SuppressWarnings("rawtypes")
   private class TestBuilder extends Value.BaseBuilder, TestBuilder> {
     TestBuilder() {
       super(ValueType.LIST);
     }
 
+    @SuppressWarnings({"unchecked", "rawtypes"})
     @Override
     public Value build() {
       return new Value(this) {
@@ -197,6 +199,7 @@ public void testGet() throws Exception {
   @Test
   public void testToBuilder() throws Exception {
     Set content = Collections.singleton("bla");
+    @SuppressWarnings("rawtypes")
     ValueBuilder builder = new TestBuilder();
     builder.meaning(1).set(content).indexed(true);
     Value value = builder.build();
diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
index 0e7d6bbbf3ea..abd912aacd23 100644
--- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -24,10 +24,10 @@
 import com.google.gcloud.storage.BlobId;
 import com.google.gcloud.storage.BlobInfo;
 import com.google.gcloud.storage.BlobReadChannel;
-import com.google.gcloud.storage.CopyWriter;
 import com.google.gcloud.storage.BlobWriteChannel;
 import com.google.gcloud.storage.Bucket;
 import com.google.gcloud.storage.BucketInfo;
+import com.google.gcloud.storage.CopyWriter;
 import com.google.gcloud.storage.Storage;
 import com.google.gcloud.storage.Storage.ComposeRequest;
 import com.google.gcloud.storage.Storage.CopyRequest;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index d874f99ebb4c..5e39bce5ed94 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -371,8 +371,10 @@ public BatchResponse batch(BatchRequest request) throws StorageException {
     List>>> partitionedToDelete =
         Lists.partition(request.toDelete, MAX_BATCH_DELETES);
     Iterator>>> iterator = partitionedToDelete.iterator();
-    BatchRequest chunkRequest = new BatchRequest(iterator.hasNext() ? iterator.next() :
-        ImmutableList.>>of(), request.toUpdate, request.toGet);
+    BatchRequest chunkRequest = new BatchRequest(
+        iterator.hasNext()
+            ? iterator.next() : ImmutableList.>>of(),
+        request.toUpdate, request.toGet);
     BatchResponse response = batchChunk(chunkRequest);
     Map> deletes =
         Maps.newHashMapWithExpectedSize(request.toDelete.size());
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java
index 503cad361e29..6bbc3843f97e 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java
@@ -18,8 +18,8 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.gcloud.storage.Blob.BlobSourceOption.toSourceOptions;
 import static com.google.gcloud.storage.Blob.BlobSourceOption.toGetOptions;
+import static com.google.gcloud.storage.Blob.BlobSourceOption.toSourceOptions;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannelImpl.java
index 8c3254a28d44..95656985043f 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannelImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannelImpl.java
@@ -37,7 +37,6 @@
  */
 class BlobWriteChannelImpl implements BlobWriteChannel {
 
-  private static final long serialVersionUID = 8675286882724938737L;
   private static final int MIN_CHUNK_SIZE = 256 * 1024;
   private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE;
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java
index 54e60cc006dc..38a767508356 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -24,8 +24,8 @@
 import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.Iterators;
-import com.google.gcloud.PageImpl;
 import com.google.gcloud.Page;
+import com.google.gcloud.PageImpl;
 import com.google.gcloud.spi.StorageRpc;
 import com.google.gcloud.storage.Storage.BlobGetOption;
 import com.google.gcloud.storage.Storage.BlobTargetOption;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
index a8cc4a0f32d8..62fbf9c6521f 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -217,6 +217,7 @@ private void readObject(ObjectInputStream in) throws IOException,
       rule = new JacksonFactory().fromString(in.readUTF(), Rule.class);
     }
 
+    @Override
     Rule toPb() {
       return rule;
     }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
index 6e1c33b137fd..23c3c19a6676 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
@@ -25,15 +25,14 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
-import com.google.gcloud.Service;
 import com.google.gcloud.Page;
+import com.google.gcloud.Service;
 import com.google.gcloud.spi.StorageRpc;
 import com.google.gcloud.spi.StorageRpc.Tuple;
 
 import java.io.InputStream;
 import java.io.Serializable;
 import java.net.URL;
-import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index e31c4073548e..d1535c92dfdb 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -522,8 +522,8 @@ private  List> transformBatch
       if (exception != null) {
         response.add(new BatchResponse.Result(exception));
       } else {
-        response.add(object != null ?
-            BatchResponse.Result.of(transform.apply(object)) : BatchResponse.Result.empty());
+        response.add(object != null
+            ? BatchResponse.Result.of(transform.apply(object)) : BatchResponse.Result.empty());
       }
     }
     return response;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java
index 2ad0950aa6aa..9ec743c079e7 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java
@@ -89,11 +89,13 @@ private StorageOptions(Builder builder) {
     pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   protected StorageFactory defaultServiceFactory() {
     return DefaultStorageFactory.INSTANCE;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   protected StorageRpcFactory defaultRpcFactory() {
     return DefaultStorageRpcFactory.INSTANCE;
@@ -118,6 +120,7 @@ public static StorageOptions defaultInstance() {
     return builder().build();
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public Builder toBuilder() {
     return new Builder(this);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java
index f99fe893d0d9..d943715580c9 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobReadChannelImplTest.java
@@ -57,7 +57,7 @@ public class BlobReadChannelImplTest {
   private BlobReadChannelImpl reader;
 
   @Before
-  public void setUp() throws IOException, InterruptedException {
+  public void setUp() {
     rpcFactoryMock = createMock(StorageRpcFactory.class);
     storageRpcMock = createMock(StorageRpc.class);
     expect(rpcFactoryMock.create(anyObject(StorageOptions.class))).andReturn(storageRpcMock);
@@ -144,7 +144,7 @@ public void testSeek() throws IOException {
   }
 
   @Test
-  public void testClose() throws IOException {
+  public void testClose() {
     replay(storageRpcMock);
     reader = new BlobReadChannelImpl(options, BLOB_ID, EMPTY_RPC_OPTIONS);
     assertTrue(reader.isOpen());
@@ -176,9 +176,9 @@ public void testReadGenerationChanged() throws IOException {
     ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
     expect(storageRpcMock.read(blobId.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
         .andReturn(StorageRpc.Tuple.of("etag1", firstResult));
-    expect(storageRpcMock.read(
-        blobId.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_SIZE))
-            .andReturn(StorageRpc.Tuple.of("etag2", firstResult));
+    expect(
+        storageRpcMock.read(blobId.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE,
+            DEFAULT_CHUNK_SIZE)).andReturn(StorageRpc.Tuple.of("etag2", secondResult));
     replay(storageRpcMock);
     reader.read(firstReadBuffer);
     try {
@@ -192,7 +192,7 @@ public void testReadGenerationChanged() throws IOException {
   }
 
   @Test
-  public void testSaveAndRestore() throws IOException, ClassNotFoundException {
+  public void testSaveAndRestore() throws IOException {
     byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE);
     byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE);
     ByteBuffer firstReadBuffer = ByteBuffer.allocate(42);
@@ -216,6 +216,7 @@ public void testSaveAndRestore() throws IOException, ClassNotFoundException {
   public void testStateEquals() {
     replay(storageRpcMock);
     reader = new BlobReadChannelImpl(options, BLOB_ID, EMPTY_RPC_OPTIONS);
+    @SuppressWarnings("resource") // avoid closing when you don't want partial writes to GCS
     BlobReadChannel secondReader = new BlobReadChannelImpl(options, BLOB_ID, EMPTY_RPC_OPTIONS);
     RestorableState state = reader.capture();
     RestorableState secondState = secondReader.capture();
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java
index 02e325716c8b..3998ae554327 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java
@@ -155,7 +155,7 @@ public void testCopyTo() throws Exception {
   public void testCopyToBlobId() throws Exception {
     BlobId targetId = BlobId.of("bt", "nt");
     CopyWriter copyWriter = createMock(CopyWriter.class);
-    BlobInfo target = BLOB_INFO.builder(targetId).build();
+    BlobInfo target = BlobInfo.builder(targetId).build();
     Capture capturedCopyRequest = Capture.newInstance();
     expect(storage.copy(capture(capturedCopyRequest))).andReturn(copyWriter);
     replay(storage);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java
index 6faa36173ab9..5d44fcb5f320 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java
@@ -65,7 +65,7 @@ public class BlobWriteChannelImplTest {
   private BlobWriteChannelImpl writer;
 
   @Before
-  public void setUp() throws IOException, InterruptedException {
+  public void setUp() {
     rpcFactoryMock = createMock(StorageRpcFactory.class);
     storageRpcMock = createMock(StorageRpc.class);
     expect(rpcFactoryMock.create(anyObject(StorageOptions.class)))
@@ -234,6 +234,7 @@ public void testStateEquals() {
         .times(2);
     replay(storageRpcMock);
     writer = new BlobWriteChannelImpl(options, BLOB_INFO, EMPTY_RPC_OPTIONS);
+    @SuppressWarnings("resource") // avoid closing when you don't want partial writes to GCS
     BlobWriteChannel writer2 = new BlobWriteChannelImpl(options, BLOB_INFO, EMPTY_RPC_OPTIONS);
     RestorableState state = writer.capture();
     RestorableState state2 = writer2.capture();
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java
index e0de2b77a899..b705685a04b1 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java
@@ -18,7 +18,6 @@
 
 import static com.google.gcloud.storage.Acl.Project.ProjectRole.VIEWERS;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
 import com.google.api.services.storage.model.Bucket.Lifecycle.Rule;
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java
index 5d8fc5a9dffc..596f43a3c87a 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java
@@ -27,8 +27,8 @@
 import static org.junit.Assert.assertTrue;
 
 import com.google.common.collect.ImmutableList;
-import com.google.gcloud.PageImpl;
 import com.google.gcloud.Page;
+import com.google.gcloud.PageImpl;
 import com.google.gcloud.storage.BatchResponse.Result;
 
 import org.easymock.Capture;
@@ -153,8 +153,8 @@ public void testGetAll() throws Exception {
     for (BlobInfo info : BLOB_INFO_RESULTS) {
       batchResultList.add(new Result<>(info));
     }
-    BatchResponse response =
-        new BatchResponse(Collections.EMPTY_LIST, Collections.EMPTY_LIST, batchResultList);
+    BatchResponse response = new BatchResponse(Collections.>emptyList(),
+        Collections.>emptyList(), batchResultList);
     expect(storage.apply(capture(capturedBatchRequest))).andReturn(response);
     replay(storage);
     List blobs = bucket.get("n1", "n2", "n3");
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java
index 0fcdb744c244..f14ee934638f 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java
@@ -21,8 +21,8 @@
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.verify;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.gcloud.RestorableState;
@@ -33,10 +33,9 @@
 
 import org.easymock.EasyMock;
 import org.junit.After;
-import org.junit.Test;
 import org.junit.Before;
+import org.junit.Test;
 
-import java.io.IOException;
 import java.util.Map;
 
 public class CopyWriterTest {
@@ -64,7 +63,7 @@ public class CopyWriterTest {
   private CopyWriter copyWriter;
 
   @Before
-  public void setUp() throws IOException, InterruptedException {
+  public void setUp() {
     rpcFactoryMock = createMock(StorageRpcFactory.class);
     storageRpcMock = createMock(StorageRpc.class);
     expect(rpcFactoryMock.create(anyObject(StorageOptions.class)))
@@ -105,7 +104,7 @@ public void testRewriteMultipleRequests() {
   }
 
   @Test
-  public void testSaveAndRestore() throws IOException {
+  public void testSaveAndRestore() {
     EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE);
     EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE_DONE);
     EasyMock.replay(storageRpcMock);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
index ed58690efde7..05055e9c3ca9 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
@@ -41,7 +41,6 @@
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
 import java.net.URL;
 import java.net.URLConnection;
 import java.nio.ByteBuffer;
@@ -53,7 +52,6 @@
 import java.util.Random;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -77,7 +75,7 @@ public static void beforeClass() {
 
   @AfterClass
   public static void afterClass()
-      throws ExecutionException, TimeoutException, InterruptedException {
+throws ExecutionException, InterruptedException {
     if (storage != null && !RemoteGcsHelper.forceDelete(storage, BUCKET, 5, TimeUnit.SECONDS)) {
       if (log.isLoggable(Level.WARNING)) {
         log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET);
@@ -154,7 +152,7 @@ public void testCreateEmptyBlob() {
   }
 
   @Test
-  public void testCreateBlobStream() throws UnsupportedEncodingException {
+  public void testCreateBlobStream() {
     String blobName = "test-create-blob-stream";
     BlobInfo blob = BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).build();
     ByteArrayInputStream stream = new ByteArrayInputStream(BLOB_STRING_CONTENT.getBytes(UTF_8));
@@ -185,7 +183,7 @@ public void testCreateBlobFail() {
   }
 
   @Test
-  public void testCreateBlobMd5Fail() throws UnsupportedEncodingException {
+  public void testCreateBlobMd5Fail() {
     String blobName = "test-create-blob-md5-fail";
     BlobInfo blob = BlobInfo.builder(BUCKET, blobName)
         .contentType(CONTENT_TYPE)
@@ -809,7 +807,7 @@ public void testReadChannelFailUpdatedGeneration() throws IOException {
       readBytes = ByteBuffer.allocate(chunkSize);
       reader.read(readBytes);
       fail("StorageException was expected");
-    } catch(StorageException ex) {
+    } catch (StorageException ex) {
       StringBuilder messageBuilder = new StringBuilder();
       messageBuilder.append("Blob ").append(blob.blobId()).append(" was updated while reading");
       assertEquals(messageBuilder.toString(), ex.getMessage());
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
index 256f5fc23f9a..af779512e0e8 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -119,6 +119,7 @@ public void testReadChannelState() throws IOException, ClassNotFoundException {
     assertEquals(state, deserializedState);
     assertEquals(state.hashCode(), deserializedState.hashCode());
     assertEquals(state.toString(), deserializedState.toString());
+    reader.close();
   }
 
   @Test
@@ -127,6 +128,7 @@ public void testWriteChannelState() throws IOException, ClassNotFoundException {
         .projectId("p2")
         .retryParams(RetryParams.defaultInstance())
         .build();
+    @SuppressWarnings("resource") // avoid closing when you don't want partial writes to GCS
     BlobWriteChannelImpl writer = new BlobWriteChannelImpl(
         options, BlobInfo.builder(BlobId.of("b", "n")).build(), "upload-id");
     RestorableState state = writer.capture();
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
index 42671d37c60c..dea635c3d264 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
@@ -251,7 +251,7 @@ public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpec
   }
 
   @Before
-  public void setUp() throws IOException, InterruptedException {
+  public void setUp() {
     rpcFactoryMock = EasyMock.createMock(StorageRpcFactory.class);
     storageRpcMock = EasyMock.createMock(StorageRpc.class);
     EasyMock.expect(rpcFactoryMock.create(EasyMock.anyObject(StorageOptions.class)))
@@ -363,7 +363,7 @@ public void testCreateBlobWithOptions() throws IOException {
   }
 
   @Test
-  public void testCreateBlobFromStream() throws IOException {
+  public void testCreateBlobFromStream() {
     ByteArrayInputStream fileStream = new ByteArrayInputStream(BLOB_CONTENT);
     BlobInfo.Builder infoBuilder = BLOB_INFO1.toBuilder();
     BlobInfo infoWithHashes = infoBuilder.md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build();

From d0eec713a4c838c67b3421b87eefa67500947e5d Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Thu, 10 Dec 2015 08:59:05 -0800
Subject: [PATCH 92/94] Add generic type to ServiceOption definition + minor
 fixes

---
 .../src/main/java/com/google/gcloud/PageImpl.java    |  3 +--
 .../main/java/com/google/gcloud/ServiceOptions.java  | 12 +++---------
 .../java/com/google/gcloud/datastore/ValueTest.java  |  2 +-
 .../gcloud/storage/BlobWriteChannelImplTest.java     |  3 ++-
 .../com/google/gcloud/storage/ITStorageTest.java     |  3 +--
 .../com/google/gcloud/storage/SerializationTest.java |  3 ++-
 6 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java b/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java
index 97c2931e0cfb..1c7a61ec471f 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/PageImpl.java
@@ -71,10 +71,9 @@ public PageImpl(NextPageFetcher pageFetcher, String cursor, Iterable resul
     this.results = results;
   }
 
-  @SuppressWarnings("unchecked")
   @Override
   public Iterable values() {
-    return results == null ? Collections.EMPTY_LIST : results;
+    return results == null ? Collections.emptyList() : results;
   }
 
   @Override
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
index 9e5d6730f551..862d3f7b533d 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -53,12 +53,8 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-@SuppressWarnings("rawtypes")
-public abstract class ServiceOptions<
-    ServiceT extends Service,
-    ServiceRpcT,
-    OptionsT extends ServiceOptions>
-    implements Serializable {
+public abstract class ServiceOptions, ServiceRpcT,
+    OptionsT extends ServiceOptions> implements Serializable {
 
   private static final String DEFAULT_HOST = "https://www.googleapis.com";
   private static final long serialVersionUID = 1203687993961393350L;
@@ -154,9 +150,7 @@ private Object readResolve() throws ObjectStreamException {
     }
   }
 
-  protected abstract static class Builder<
-      ServiceT extends Service,
-      ServiceRpcT,
+  protected abstract static class Builder, ServiceRpcT,
       OptionsT extends ServiceOptions,
       B extends Builder> {
 
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
index 3867e194d060..891668990f66 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
@@ -66,7 +66,7 @@ private class TestBuilder extends Value.BaseBuilder, TestBuilder
       super(ValueType.LIST);
     }
 
-    @SuppressWarnings({"unchecked", "rawtypes"})
+    @SuppressWarnings({"unchecked"})
     @Override
     public Value build() {
       return new Value(this) {
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java
index 5d44fcb5f320..c2107ff13998 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobWriteChannelImplTest.java
@@ -234,7 +234,8 @@ public void testStateEquals() {
         .times(2);
     replay(storageRpcMock);
     writer = new BlobWriteChannelImpl(options, BLOB_INFO, EMPTY_RPC_OPTIONS);
-    @SuppressWarnings("resource") // avoid closing when you don't want partial writes to GCS
+    // avoid closing when you don't want partial writes to GCS upon failure
+    @SuppressWarnings("resource")
     BlobWriteChannel writer2 = new BlobWriteChannelImpl(options, BLOB_INFO, EMPTY_RPC_OPTIONS);
     RestorableState state = writer.capture();
     RestorableState state2 = writer2.capture();
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
index 05055e9c3ca9..22b0a35c0620 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
@@ -74,8 +74,7 @@ public static void beforeClass() {
   }
 
   @AfterClass
-  public static void afterClass()
-throws ExecutionException, InterruptedException {
+  public static void afterClass() throws ExecutionException, InterruptedException {
     if (storage != null && !RemoteGcsHelper.forceDelete(storage, BUCKET, 5, TimeUnit.SECONDS)) {
       if (log.isLoggable(Level.WARNING)) {
         log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
index af779512e0e8..555e231f7f0e 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -128,7 +128,8 @@ public void testWriteChannelState() throws IOException, ClassNotFoundException {
         .projectId("p2")
         .retryParams(RetryParams.defaultInstance())
         .build();
-    @SuppressWarnings("resource") // avoid closing when you don't want partial writes to GCS
+    // avoid closing when you don't want partial writes to GCS upon failure
+    @SuppressWarnings("resource")
     BlobWriteChannelImpl writer = new BlobWriteChannelImpl(
         options, BlobInfo.builder(BlobId.of("b", "n")).build(), "upload-id");
     RestorableState state = writer.capture();

From f22e41fd26a4d5aaacd3fc4e32083a060d018f9b Mon Sep 17 00:00:00 2001
From: Ajay Kannan 
Date: Thu, 10 Dec 2015 16:52:42 -0800
Subject: [PATCH 93/94] Specify HTTP Transport in DefaultDatastoreRpc

---
 .../src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
index 028027f4bc33..4771895eec99 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
@@ -69,6 +69,7 @@ public DefaultDatastoreRpc(DatastoreOptions options) {
     String normalizedHost = normalizeHost(options.host());
     client = DatastoreFactory.get().create(
         new Builder()
+            .transport(options.httpTransportFactory().create())
             .dataset(options.projectId())
             .host(normalizedHost)
             .initializer(options.httpRequestInitializer())

From a08e7bb7e091e8871097925bb54cde57082768b0 Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Fri, 18 Dec 2015 10:00:32 +0100
Subject: [PATCH 94/94] Change poms and code to match with master udpates

---
 gcloud-java-bigquery/pom.xml                               | 2 +-
 .../java/com/google/gcloud/bigquery/BigQueryImplTest.java  | 7 +++----
 .../java/com/google/gcloud/bigquery/SerializationTest.java | 4 ++--
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/gcloud-java-bigquery/pom.xml b/gcloud-java-bigquery/pom.xml
index f4d459ab82f7..7b749097869f 100644
--- a/gcloud-java-bigquery/pom.xml
+++ b/gcloud-java-bigquery/pom.xml
@@ -11,7 +11,7 @@
   
     com.google.gcloud
     gcloud-java-pom
-    0.0.11-SNAPSHOT
+    0.1.1-SNAPSHOT
   
   
     gcloud-java-bigquery
diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java
index c9c46e0a632d..85c67b36b1c4 100644
--- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java
+++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java
@@ -260,7 +260,6 @@ public void setUp() {
     EasyMock.replay(rpcFactoryMock);
     options = BigQueryOptions.builder()
         .projectId(PROJECT)
-        .authCredentials(AuthCredentials.noCredentials())
         .serviceRpcFactory(rpcFactoryMock)
         .build();
   }
@@ -1011,7 +1010,7 @@ public void testRetryableException() {
         .andThrow(new BigQueryException(500, "InternalError", true))
         .andReturn(DATASET_INFO_WITH_PROJECT.toPb());
     EasyMock.replay(bigqueryRpcMock);
-    bigquery = options.toBuilder().retryParams(RetryParams.getDefaultInstance()).build().service();
+    bigquery = options.toBuilder().retryParams(RetryParams.defaultInstance()).build().service();
     DatasetInfo dataset = bigquery.getDataset(DATASET);
     assertEquals(DATASET_INFO_WITH_PROJECT, dataset);
   }
@@ -1022,7 +1021,7 @@ public void testNonRetryableException() {
     EasyMock.expect(bigqueryRpcMock.getDataset(DATASET, EMPTY_RPC_OPTIONS))
         .andThrow(new BigQueryException(501, exceptionMessage, false));
     EasyMock.replay(bigqueryRpcMock);
-    bigquery = options.toBuilder().retryParams(RetryParams.getDefaultInstance()).build().service();
+    bigquery = options.toBuilder().retryParams(RetryParams.defaultInstance()).build().service();
     thrown.expect(BigQueryException.class);
     thrown.expectMessage(exceptionMessage);
     bigquery.getDataset(DatasetId.of(DATASET));
@@ -1034,7 +1033,7 @@ public void testRuntimeException() {
     EasyMock.expect(bigqueryRpcMock.getDataset(DATASET, EMPTY_RPC_OPTIONS))
         .andThrow(new RuntimeException(exceptionMessage));
     EasyMock.replay(bigqueryRpcMock);
-    bigquery = options.toBuilder().retryParams(RetryParams.getDefaultInstance()).build().service();
+    bigquery = options.toBuilder().retryParams(RetryParams.defaultInstance()).build().service();
     thrown.expect(BigQueryException.class);
     thrown.expectMessage(exceptionMessage);
     bigquery.getDataset(DATASET);
diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java
index bbd244367a20..8c80bddbfefb 100644
--- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java
+++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java
@@ -218,8 +218,8 @@ public void testServiceOptions() throws Exception {
 
     options = options.toBuilder()
         .projectId("p2")
-        .retryParams(RetryParams.getDefaultInstance())
-        .authCredentials(AuthCredentials.noCredentials())
+        .retryParams(RetryParams.defaultInstance())
+        .authCredentials(null)
         .build();
     serializedCopy = serializeAndDeserialize(options);
     assertEquals(options, serializedCopy);