From 6c59a54b8bdd75d899981aa10da6c4887da72b91 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 12 May 2015 14:02:30 -0700 Subject: [PATCH 1/5] add metadata update to example and fix builder user-settable fields --- .../gcloud/examples/StorageExample.java | 44 ++++++++++++++++++- .../java/com/google/gcloud/storage/Blob.java | 28 +++++++----- .../com/google/gcloud/storage/Bucket.java | 4 +- .../gcloud/storage/StorageServiceImpl.java | 7 +-- 4 files changed, 66 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index 8b1a79b37212..453576683966 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -57,7 +57,7 @@ * -Dexec.args="[] list []| info [ []]| * download [local_file]| upload []| * delete +| cp | - * compose + "} + * compose + | update_metadata [key=value]*"} * * */ @@ -336,6 +336,45 @@ public String params() { } } + private static class UpdateMetadata extends StorageAction>> { + + @Override + public void run(StorageService storage, Tuple> tuple) + throws IOException { + Blob blob = storage.get(tuple.x().bucket(), tuple.x().name()); + if (blob == null) { + System.out.println("No such object"); + return; + } + blob = blob.toBuilder().metadata(tuple.y()).build(); + System.out.println("before: " + blob); + System.out.println(storage.update(blob)); + } + + @Override + Tuple> parse(String... args) { + if (args.length < 2) { + throw new IllegalArgumentException(); + } + Blob blob = Blob.of(args[0], args[1]); + Map metadata = new HashMap<>(); + for (int i = 2; i < args.length; i++) { + int idx = args[i].indexOf('='); + if (idx < 0) { + metadata.put(args[i], ""); + } else { + metadata.put(args[i].substring(0, idx), args[i].substring(idx + 1)); + } + } + return Tuple.of(blob, metadata); + } + + @Override + public String params() { + return " [local_file]"; + } + } + static { ACTIONS.put("info", new InfoAction()); ACTIONS.put("delete", new DeleteAction()); @@ -344,6 +383,7 @@ public String params() { ACTIONS.put("download", new DownloadAction()); ACTIONS.put("cp", new CopyAction()); ACTIONS.put("compose", new ComposeAction()); + ACTIONS.put("update_metadata", new UpdateMetadata()); } public static void printUsage() { @@ -378,7 +418,7 @@ public static void main(String... args) throws Exception { args = Arrays.copyOfRange(args, 1, args.length); } if (action == null) { - System.out.println("Unrecognized action '" + args[1] + "'"); + System.out.println("Unrecognized action."); printUsage(); return; } diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java index 49d763b0f65a..ffaec524c67d 100644 --- a/src/main/java/com/google/gcloud/storage/Blob.java +++ b/src/main/java/com/google/gcloud/storage/Blob.java @@ -127,12 +127,12 @@ public Builder contentType(String contentType) { return this; } - Builder contentDisposition(String contentDisposition) { + public Builder contentDisposition(String contentDisposition) { this.contentDisposition = contentDisposition; return this; } - Builder contentLanguage(String contentLanguage) { + public Builder contentLanguage(String contentLanguage) { this.contentLanguage = contentLanguage; return this; } @@ -157,12 +157,12 @@ public Builder acl(List acl) { return this; } - public Builder owner(Acl.Entity owner) { + Builder owner(Acl.Entity owner) { this.owner = owner; return this; } - public Builder size(Long size) { + Builder size(Long size) { this.size = size; return this; } @@ -177,7 +177,7 @@ Builder selfLink(String selfLink) { return this; } - Builder md5(String md5) { + public Builder md5(String md5) { this.md5 = md5; return this; } @@ -187,7 +187,7 @@ public Builder crc32c(String crc32c) { return this; } - public Builder mediaLink(String mediaLink) { + Builder mediaLink(String mediaLink) { this.mediaLink = mediaLink; return this; } @@ -197,22 +197,22 @@ public Builder metadata(Map metadata) { return this; } - public Builder generation(Long generation) { + Builder generation(Long generation) { this.generation = generation; return this; } - public Builder metageneration(Long metageneration) { + Builder metageneration(Long metageneration) { this.metageneration = metageneration; return this; } - public Builder deleteTime(Long deleteTime) { + Builder deleteTime(Long deleteTime) { this.deleteTime = deleteTime; return this; } - public Builder updateTime(Long updateTime) { + Builder updateTime(Long updateTime) { this.updateTime = updateTime; return this; } @@ -365,7 +365,13 @@ public Builder toBuilder() { @Override public String toString() { - return MoreObjects.toStringHelper(this).add("bucket", bucket).add("name", name).toString(); + return MoreObjects.toStringHelper(this) + .add("bucket", bucket) + .add("name", name) + .add("size", size) + .add("content-type", contentType) + .add("metadata", metadata) + .toString(); } public static Blob of(String bucket, String name) { diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java index f6be9a78b2c6..7f68704f9e6a 100644 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -561,7 +561,9 @@ public Builder toBuilder() { @Override public String toString() { - return MoreObjects.toStringHelper(this).add("name", name).toString(); + return MoreObjects.toStringHelper(this) + .add("name", name) + .toString(); } public static Bucket of(String name) { diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 42299f2bc6bd..02bbb97a2fe5 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -28,6 +28,7 @@ import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.util.concurrent.Executors.callable; import com.google.api.services.storage.model.StorageObject; @@ -131,7 +132,7 @@ public com.google.api.services.storage.model.Bucket call() { try { return storageRpc.get(bucketPb, optionsMap); } catch (StorageServiceException ex) { - if (ex.code() == 404) { + if (ex.code() == HTTP_NOT_FOUND) { return null; } throw ex; @@ -151,7 +152,7 @@ public StorageObject call() { try { return storageRpc.get(storedObject, optionsMap); } catch (StorageServiceException ex) { - if (ex.code() == 404) { + if (ex.code() == HTTP_NOT_FOUND) { return null; } throw ex; @@ -328,7 +329,7 @@ public BatchResponse apply(BatchRequest batchRequest) { List> updates = transformBatchResult( toUpdate, response.updates, Blob.FROM_PB_FUNCTION); List> gets = transformBatchResult( - toGet, response.gets, Blob.FROM_PB_FUNCTION, 404); + toGet, response.gets, Blob.FROM_PB_FUNCTION, HTTP_NOT_FOUND); return new BatchResponse(deletes, updates, gets); } From 17ec594784481c17f380499f2586ff15ec6b78db Mon Sep 17 00:00:00 2001 From: ozarov Date: Tue, 12 May 2015 19:26:24 -0700 Subject: [PATCH 2/5] add serialization test --- .../java/com/google/gcloud/storage/Acl.java | 25 ++++++--- .../google/gcloud/storage/BatchRequest.java | 53 +++++++++++++------ .../google/gcloud/storage/BatchResponse.java | 34 +++++++++++- .../java/com/google/gcloud/storage/Blob.java | 16 +++++- .../com/google/gcloud/storage/Bucket.java | 14 +++++ .../java/com/google/gcloud/storage/Cors.java | 33 +++++++++++- .../com/google/gcloud/storage/ListResult.java | 18 ++++++- .../gcloud/storage/StorageServiceImpl.java | 15 +++--- .../gcloud/storage/SerializationTest.java | 47 +++++++++++++++- 9 files changed, 221 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java index 138ba4f3fa3a..b5bb685334c1 100644 --- a/src/main/java/com/google/gcloud/storage/Acl.java +++ b/src/main/java/com/google/gcloud/storage/Acl.java @@ -20,6 +20,7 @@ import com.google.api.services.storage.model.ObjectAccessControl; import java.io.Serializable; +import java.util.Objects; /** * Access Control List on for buckets or blobs. @@ -59,6 +60,19 @@ protected String value() { return value; } + @Override + public int hashCode() { + return Objects.hash(type, value); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !getClass().isAssignableFrom(obj.getClass())) { + return false; + } + return Objects.equals(toPb(), ((Entity)obj).toPb()); + } + @Override public String toString() { return toPb(); @@ -91,13 +105,12 @@ static Entity fromPb(String entity) { } } - public static class Domain extends Entity { + public static final class Domain extends Entity { private static final long serialVersionUID = -3033025857280447253L; public Domain(String domain) { super(Type.DOMAIN, domain); - } public String domain() { @@ -105,7 +118,7 @@ public String domain() { } } - public static class Group extends Entity { + public static final class Group extends Entity { private static final long serialVersionUID = -1660987136294408826L; @@ -118,7 +131,7 @@ public String email() { } } - public static class User extends Entity { + public static final class User extends Entity { private static final long serialVersionUID = 3076518036392737008L; private static final String ALL_USERS = "allUsers"; @@ -154,7 +167,7 @@ public static User ofAllAuthenticatedUsers() { } } - public static class Project extends Entity { + public static final class Project extends Entity { private static final long serialVersionUID = 7933776866530023027L; @@ -180,7 +193,7 @@ public String projectId() { } } - public static class RawEntity extends Entity { + public static final class RawEntity extends Entity { private static final long serialVersionUID = 3966205614223053950L; diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java index 9fbee876cf45..6f62d5c51ae4 100644 --- a/src/main/java/com/google/gcloud/storage/BatchRequest.java +++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java @@ -17,51 +17,56 @@ package com.google.gcloud.storage; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.google.gcloud.storage.StorageService.BlobSourceOption; import com.google.gcloud.storage.StorageService.BlobTargetOption; import java.io.Serializable; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; /** * Google storage batch request. */ -public class BatchRequest implements Serializable { +public final class BatchRequest implements Serializable { private static final long serialVersionUID = -1527992265939800345L; - private final Map toDelete; - private final Map toUpdate; - private final Map toGet; + private final Map> toDelete; + private final Map> toUpdate; + private final Map> toGet; public static class Builder { - private Map toDelete = new LinkedHashMap<>(); - private Map toUpdate = new LinkedHashMap<>(); - private Map toGet = new LinkedHashMap<>(); + private Map> toDelete = new LinkedHashMap<>(); + private Map> toUpdate = new LinkedHashMap<>(); + private Map> toGet = new LinkedHashMap<>(); private Builder() {} /** * Delete the given blob. */ - public void delete(String bucket, String blob, BlobSourceOption... options) { - toDelete.put(Blob.of(bucket, blob), options); + public Builder delete(String bucket, String blob, BlobSourceOption... options) { + toDelete.put(Blob.of(bucket, blob), Lists.newArrayList(options)); + return this; } /** * Update the given blob. */ - public void update(Blob blob, BlobTargetOption... options) { - toUpdate.put(blob, options); + public Builder update(Blob blob, BlobTargetOption... options) { + toUpdate.put(blob, Lists.newArrayList(options)); + return this; } /** * Retrieve metadata for the given blob. */ - public void get(String bucket, String blob, BlobSourceOption... options) { - toGet.put(Blob.of(bucket, blob), options); + public Builder get(String bucket, String blob, BlobSourceOption... options) { + toGet.put(Blob.of(bucket, blob), Lists.newArrayList(options)); + return this; } public BatchRequest build() { @@ -75,15 +80,31 @@ private BatchRequest(Builder builder) { toGet = ImmutableMap.copyOf(builder.toGet); } - Map toDelete() { + @Override + public int hashCode() { + return Objects.hash(toDelete, toUpdate, toGet); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof BatchRequest)) { + return false; + } + BatchRequest other = (BatchRequest) obj; + return Objects.equals(toDelete, other.toDelete) + && Objects.equals(toUpdate, other.toUpdate) + && Objects.equals(toGet, other.toGet); + } + + Map> toDelete() { return toDelete; } - Map toUpdate() { + Map> toUpdate() { return toUpdate; } - Map toGet() { + Map> toGet() { return toGet; } diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java index fab4d964e1c6..c45b0623eb9b 100644 --- a/src/main/java/com/google/gcloud/storage/BatchResponse.java +++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java @@ -21,11 +21,12 @@ import java.io.Serializable; import java.util.List; +import java.util.Objects; /** * Google Storage batch response. */ -public class BatchResponse implements Serializable { +public final class BatchResponse implements Serializable { private static final long serialVersionUID = 1057416839397037706L; @@ -78,6 +79,21 @@ public boolean failed() { return exception != null; } + @Override + public int hashCode() { + return Objects.hash(value, exception); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Result)) { + return false; + } + Result other = (Result) obj; + return Objects.equals(value, other.value) + && Objects.equals(exception, other.exception); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) @@ -99,6 +115,22 @@ static Result empty() { this.getResult = ImmutableList.copyOf(getResult); } + @Override + public int hashCode() { + return Objects.hash(deleteResult, updateResult, getResult); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof BatchResponse)) { + return false; + } + BatchResponse other = (BatchResponse) obj; + return Objects.equals(deleteResult, other.deleteResult) + && Objects.equals(updateResult, other.updateResult) + && Objects.equals(updateResult, other.updateResult); + } + /** * Returns the results for the delete operations using the request order. */ diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java index ffaec524c67d..7ef81bf2b240 100644 --- a/src/main/java/com/google/gcloud/storage/Blob.java +++ b/src/main/java/com/google/gcloud/storage/Blob.java @@ -32,13 +32,14 @@ import java.math.BigInteger; import java.util.List; import java.util.Map; +import java.util.Objects; /** * A Google Storage object. * * @see Concepts and Terminology */ -public class Blob implements Serializable { +public final class Blob implements Serializable { private static final long serialVersionUID = 2228487739943277159L; @@ -386,6 +387,19 @@ public static Builder builder(String bucket, String name) { return new Builder().bucket(bucket).name(name); } + @Override + public int hashCode() { + return Objects.hash(bucket, name); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Blob)) { + return false; + } + return Objects.equals(toPb(), ((Blob)obj).toPb()); + } + StorageObject toPb() { StorageObject storageObject = new StorageObject(); if (acl != null) { diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java index 7f68704f9e6a..c36b24c80269 100644 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -39,6 +39,7 @@ import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.List; +import java.util.Objects; /** * A Google Storage bucket. @@ -559,6 +560,19 @@ public Builder toBuilder() { .deleteRules(deleteRules); } + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Bucket)) { + return false; + } + return Objects.equals(toPb(), ((Bucket) obj).toPb()); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java index 366c8d9223bd..9cedbc0d3b4c 100644 --- a/src/main/java/com/google/gcloud/storage/Cors.java +++ b/src/main/java/com/google/gcloud/storage/Cors.java @@ -29,6 +29,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.List; +import java.util.Objects; /** * Cross-Origin Resource Sharing (CORS) configuration for a bucket. @@ -60,7 +61,7 @@ public enum Method { ANY, GET, HEAD, PUT, POST, DELETE } - public static class Origin implements Serializable { + public static final class Origin implements Serializable { private static final long serialVersionUID = -4447958124895577993L; private static final String ANY_URI = "*"; @@ -91,6 +92,19 @@ public static Origin of(String value) { return new Origin(value); } + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Origin)) { + return false; + } + return value.equals(((Origin)obj).value); + } + @Override public String toString() { return value(); @@ -166,6 +180,23 @@ public Builder toBuilder() { .responseHeaders(responseHeaders); } + @Override + public int hashCode() { + return Objects.hash(maxAgeSeconds, methods, origins, responseHeaders); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Cors)) { + return false; + } + Cors other = (Cors) obj; + return Objects.equals(maxAgeSeconds, other.maxAgeSeconds) + && Objects.equals(methods, other.methods) + && Objects.equals(origins, other.origins) + && Objects.equals(responseHeaders, other.responseHeaders); + } + public static Builder builder() { return new Builder(); } diff --git a/src/main/java/com/google/gcloud/storage/ListResult.java b/src/main/java/com/google/gcloud/storage/ListResult.java index 180e5f7a4155..dd843020376e 100644 --- a/src/main/java/com/google/gcloud/storage/ListResult.java +++ b/src/main/java/com/google/gcloud/storage/ListResult.java @@ -18,11 +18,12 @@ import java.io.Serializable; import java.util.Iterator; +import java.util.Objects; /** * Google Cloud storage list result. */ -public class ListResult implements Iterable, Serializable { +public final class ListResult implements Iterable, Serializable { private static final long serialVersionUID = -6937287874908527950L; @@ -42,4 +43,19 @@ public String nextPageCursor() { public Iterator iterator() { return results.iterator(); } + + @Override + public int hashCode() { + return Objects.hash(cursor, results); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ListResult)) { + return false; + } + ListResult other = (ListResult) obj; + return Objects.equals(cursor, other.cursor) + && Objects.equals(results, other.results); + } } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index 02bbb97a2fe5..1f66401105d0 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -302,24 +302,27 @@ public byte[] call() { public BatchResponse apply(BatchRequest batchRequest) { List>> toDelete = Lists.newArrayListWithCapacity(batchRequest.toDelete().size()); - for (Map.Entry entry : batchRequest.toDelete().entrySet()) { + for (Map.Entry> entry : batchRequest.toDelete().entrySet()) { Blob blob = entry.getKey(); - Map optionsMap = optionMap(blob, entry.getValue()); + Map optionsMap = + optionMap(blob.generation(), blob.metageneration(), entry.getValue()); StorageObject storageObject = blob.toPb(); toDelete.add(Tuple.>of(storageObject, optionsMap)); } List>> toUpdate = Lists.newArrayListWithCapacity(batchRequest.toUpdate().size()); - for (Map.Entry entry : batchRequest.toUpdate().entrySet()) { + for (Map.Entry> entry : batchRequest.toUpdate().entrySet()) { Blob blob = entry.getKey(); - Map optionsMap = optionMap(blob, entry.getValue()); + Map optionsMap = + optionMap(blob.generation(), blob.metageneration(), entry.getValue()); toUpdate.add(Tuple.>of(blob.toPb(), optionsMap)); } List>> toGet = Lists.newArrayListWithCapacity(batchRequest.toGet().size()); - for (Map.Entry entry : batchRequest.toGet().entrySet()) { + for (Map.Entry> entry : batchRequest.toGet().entrySet()) { Blob blob = entry.getKey(); - Map optionsMap = optionMap(blob, entry.getValue()); + Map optionsMap = + optionMap(blob.generation(), blob.metageneration(), entry.getValue()); toGet.add(Tuple.>of(blob.toPb(), optionsMap)); } StorageRpc.BatchResponse response = diff --git a/src/test/java/com/google/gcloud/storage/SerializationTest.java b/src/test/java/com/google/gcloud/storage/SerializationTest.java index 7e3cb3e89258..365462d3f69a 100644 --- a/src/test/java/com/google/gcloud/storage/SerializationTest.java +++ b/src/test/java/com/google/gcloud/storage/SerializationTest.java @@ -17,9 +17,11 @@ package com.google.gcloud.storage; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; import com.google.gcloud.AuthCredentials; import com.google.gcloud.RetryParams; +import com.google.gcloud.storage.Acl.Project.ProjectRole; import org.junit.Test; @@ -28,9 +30,40 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.Collections; public class SerializationTest { + private static final Acl.Domain ACL_DOMAIN = new Acl.Domain("domain"); + private static final Acl.Group ACL_GROUP = new Acl.Group("group"); + private static final Acl.Project ACL_PROJECT_ = new Acl.Project(ProjectRole.VIEWERS, "pid"); + private static final Acl.User ACL_USER = new Acl.User("user"); + private static final Acl.RawEntity ACL_RAW = new Acl.RawEntity("raw"); + private static final Blob BLOB = Blob.of("b", "n"); + private static final Bucket BUCKET = Bucket.of("b"); + private static final Cors.Origin ORIGIN = Cors.Origin.any(); + private static final Cors CORS = + Cors.builder().maxAgeSeconds(1).origins(Collections.singleton(ORIGIN)).build(); + private static final BatchRequest BATCH_REQUEST = BatchRequest.builder().delete("B", "N").build(); + private static final BatchResponse BATCH_RESPONSE = new BatchResponse( + Collections.singletonList(new BatchResponse.Result<>(true)), + Collections.>emptyList(), + Collections.>emptyList()); + private static final ListResult LIST_RESULT = + new ListResult<>("c", Collections.singletonList(Blob.of("b", "n"))); + private static StorageService.BlobListOption BLOB_LIST_OPTIONS = + StorageService.BlobListOption.maxResults(100); + private static StorageService.BlobSourceOption BLOB_SOURCE_OPTIONS = + StorageService.BlobSourceOption.generationMatch(1); + private static StorageService.BlobTargetOption BLOB_TARGET_OPTIONS = + StorageService.BlobTargetOption.generationMatch(); + private static StorageService.BucketListOption BUCKET_LIST_OPTIONS = + StorageService.BucketListOption.prefix("bla"); + private static StorageService.BucketSourceOption BUCKET_SOURCE_OPTIONS = + StorageService.BucketSourceOption.metagenerationMatch(1); + private static StorageService.BucketTargetOption BUCKET_TARGET_OPTIONS = + StorageService.BucketTargetOption.metagenerationNotMatch(); @Test public void testServiceOptions() throws Exception { @@ -52,8 +85,18 @@ public void testServiceOptions() throws Exception { } @Test - public void testTypes() throws Exception { - // todo: implement + public void testModelAndRequests() throws Exception { + Serializable[] objects = {ACL_DOMAIN, ACL_GROUP, ACL_PROJECT_, ACL_USER, ACL_RAW, BLOB, BUCKET, + ORIGIN, CORS, BATCH_REQUEST,BATCH_RESPONSE, LIST_RESULT, BLOB_LIST_OPTIONS, + BLOB_SOURCE_OPTIONS, BLOB_TARGET_OPTIONS, BUCKET_LIST_OPTIONS, BUCKET_SOURCE_OPTIONS, + BUCKET_TARGET_OPTIONS}; + for (Serializable obj : objects) { + Object copy = serializeAndDeserialize(obj); + assertEquals(obj, obj); + assertEquals(obj, copy); + assertNotSame(obj, copy); + assertEquals(copy, copy); + } } @SuppressWarnings("unchecked") From 4b8bd324d670988430c9634ce9271cc87cb075b7 Mon Sep 17 00:00:00 2001 From: ozarov Date: Tue, 12 May 2015 22:02:20 -0700 Subject: [PATCH 3/5] add comments to example --- .../gcloud/examples/StorageExample.java | 133 ++++++++++++++---- .../google/gcloud/storage/BatchResponse.java | 2 +- 2 files changed, 108 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index 453576683966..05f1f9336107 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -46,7 +46,7 @@ /** * An example of using the Google Cloud Storage. *

- * This example demonstrates a simple/typical usage. + * This example demonstrates a simple/typical storage usage. *

* Steps needed for running the example: *

    @@ -60,6 +60,11 @@ * compose + | update_metadata [key=value]*"} * *
+ * + * The first parameter is an optional project_id (logged-in project will be used if not supplied). + * Second parameter is a Storage operation (list, delete, compose,...) to demonstrate the its + * usage. Any other arguments are specific to the operation. + * See each action's run method for the specific Storage interaction. */ public class StorageExample { @@ -112,24 +117,34 @@ public String params() { } } + /** + * This class demonstrates how to retrieve Bucket or Blob metadata. + * If more than one blob is supplied a Batch operation would be used to get all blobs metadata + * in a single RPC. + */ private static class InfoAction extends BlobsAction { @Override public void run(StorageService storage, Blob... blobs) { - - if (blobs.length == 1) { if (blobs[0].name().isEmpty()) { - System.out.println(storage.get(blobs[0].bucket())); + // get Bucket + Bucket bucket = storage.get(blobs[0].bucket()); + System.out.println("Bucket info: " + bucket); } else { - System.out.println(storage.get(blobs[0].bucket(), blobs[0].name())); + // get Blob + Blob blob = storage.get(blobs[0].bucket(), blobs[0].name()); + System.out.println("Blob info: " + blob); } } else { + // use batch to get multiple blobs. BatchRequest.Builder batch = BatchRequest.builder(); for (Blob blob : blobs) { batch.get(blob.bucket(), blob.name()); } BatchResponse response = storage.apply(batch.build()); - System.out.println(response.gets()); + for (BatchResponse.Result result : response.gets()) { + System.out.println(result.get()); + } } } @@ -147,22 +162,41 @@ public String params() { } } + /** + * This class demonstrates how to delete a blob. + * If more than one blob is supplied a Batch operation would be used to delete all requested + * blobs in a single RPC. + */ private static class DeleteAction extends BlobsAction { @Override public void run(StorageService storage, Blob... blobs) { if (blobs.length == 1) { - System.out.println(storage.delete(blobs[0].bucket(), blobs[0].name())); + boolean wasDeleted = storage.delete(blobs[0].bucket(), blobs[0].name()); + if (wasDeleted) { + System.out.println("Blob " + blobs[0] + " was deleted"); + } } else { + // use batch operation BatchRequest.Builder batch = BatchRequest.builder(); for (Blob blob : blobs) { batch.delete(blob.bucket(), blob.name()); } + int index = 0; BatchResponse response = storage.apply(batch.build()); - System.out.println(response.deletes()); + for (BatchResponse.Result result : response.deletes()) { + if (result.get()) { + // request order is maintained + System.out.println("Blob " + blobs[index] + " was deleted"); + } + index++; + } } } } + /** + * This class demonstrates how to list buckets or a bucket's blobs. + */ private static class ListAction extends StorageAction { @Override @@ -179,10 +213,12 @@ String parse(String... args) { @Override public void run(StorageService storage, String bucket) { if (bucket == null) { + // list buckets for (Bucket b : storage.list()) { System.out.println(b); } } else { + // list a bucket's blobs for (Blob b : storage.list(bucket)) { System.out.println(b); } @@ -195,13 +231,22 @@ public String params() { } } + /** + * This class demonstrates how to create a new Blob or to update its content. + */ private static class UploadAction extends StorageAction> { @Override public void run(StorageService storage, Tuple tuple) throws Exception { - if (Files.size(tuple.x()) > 1024) { - try (BlobWriteChannel writer = storage.writer(tuple.y())) { + run(storage, tuple.x(), tuple.y()); + } + + private void run(StorageService storage, Path uploadFrom, Blob blob) throws IOException { + if (Files.size(uploadFrom) > 1_000_000) { + // When content is not available or large (1MB or more) it is recommended + // to write it in chunks via the blob's channel writer. + try (BlobWriteChannel writer = storage.writer(blob)) { byte[] buffer = new byte[1024]; - try (InputStream input = Files.newInputStream(tuple.x())) { + try (InputStream input = Files.newInputStream(uploadFrom)) { int limit; while ((limit = input.read(buffer)) >= 0) { try { @@ -213,9 +258,11 @@ public void run(StorageService storage, Tuple tuple) throws Exceptio } } } else { - byte[] bytes = Files.readAllBytes(tuple.x()); - System.out.println(storage.create(tuple.y(), bytes)); + byte[] bytes = Files.readAllBytes(uploadFrom); + // create the blob in one request. + storage.create(blob, bytes); } + System.out.println("Blob was created"); } @Override @@ -235,22 +282,35 @@ public String params() { } } + /** + * This class demonstrates how read a blob's content. + * The example will dump the content to a local file if one was given or write + * it to stdout otherwise. + */ private static class DownloadAction extends StorageAction> { @Override public void run(StorageService storage, Tuple tuple) throws IOException { - Blob blob = storage.get(tuple.x().bucket(), tuple.x().name()); + run(storage, tuple.x().bucket(), tuple.x().name(), tuple.y()); + } + + private void run(StorageService storage, String bucket, String blobName, Path downloadTo) + throws IOException { + Blob blob = storage.get(bucket, blobName); if (blob == null) { System.out.println("No such object"); return; } PrintStream writeTo = System.out; - if (tuple.y() != null) { - writeTo = new PrintStream(new FileOutputStream(tuple.y().toFile())); + if (downloadTo != null) { + writeTo = new PrintStream(new FileOutputStream(downloadTo.toFile())); } - if (blob.size() < 1024) { - writeTo.write(storage.load(blob.bucket(), blob.name())); + if (blob.size() < 1_000_000) { + // Blob is small read all its content in one request + byte[] content = storage.load(blob.bucket(), blob.name()); + writeTo.write(content); } else { + // When Blob size is big or unknown use the blob's channel reader. try (BlobReadChannel reader = storage.reader(blob.bucket(), blob.name())) { WritableByteChannel channel = Channels.newChannel(writeTo); ByteBuffer bytes = ByteBuffer.allocate(64 * 1024); @@ -261,7 +321,7 @@ public void run(StorageService storage, Tuple tuple) throws IOExcept } } } - if (tuple.y() == null) { + if (downloadTo == null) { writeTo.println(); } else { writeTo.close(); @@ -291,10 +351,16 @@ public String params() { } } + /** + * This class demonstrates how to use the copy command. + * + * @see Object copy + */ private static class CopyAction extends StorageAction { @Override public void run(StorageService storage, CopyRequest request) { - System.out.println(storage.copy(request)); + Blob copiedBlob = storage.copy(request); + System.out.println("Copied " + copiedBlob); } @Override @@ -311,10 +377,16 @@ public String params() { } } + /** + * This class demonstrates how to use the compose command. + * + * @see Object compose + */ private static class ComposeAction extends StorageAction { @Override public void run(StorageService storage, ComposeRequest request) { - System.out.println(storage.compose(request)); + Blob composedBlob = storage.compose(request); + System.out.println("Composed " + composedBlob); } @Override @@ -336,19 +408,28 @@ public String params() { } } + /** + * This class demonstrates how to update a blob's metadata. + * + * @see Object compose + */ private static class UpdateMetadata extends StorageAction>> { @Override public void run(StorageService storage, Tuple> tuple) throws IOException { - Blob blob = storage.get(tuple.x().bucket(), tuple.x().name()); + run(storage, tuple.x().bucket(), tuple.x().name(), tuple.y()); + } + + private void run(StorageService storage, String bucket, String blobName, + Map metadata) { + Blob blob = storage.get(bucket, blobName); if (blob == null) { System.out.println("No such object"); return; } - blob = blob.toBuilder().metadata(tuple.y()).build(); - System.out.println("before: " + blob); - System.out.println(storage.update(blob)); + blob = storage.update(blob.toBuilder().metadata(metadata).build()); + System.out.println("Updated " + blob); } @Override @@ -357,7 +438,7 @@ Tuple> parse(String... args) { throw new IllegalArgumentException(); } Blob blob = Blob.of(args[0], args[1]); - Map metadata = new HashMap<>(); + Map metadata = new HashMap<>(); for (int i = 2; i < args.length; i++) { int idx = args[i].indexOf('='); if (idx < 0) { diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java index c45b0623eb9b..f0675e348f72 100644 --- a/src/main/java/com/google/gcloud/storage/BatchResponse.java +++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java @@ -58,7 +58,7 @@ public static class Result implements Serializable { * * @throws StorageServiceException if failed */ - public T result() throws StorageServiceException { + public T get() throws StorageServiceException { if (failed()) { throw failure(); } From 96b3ada3ece83a95e42489715d760a66f9494f05 Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 13 May 2015 16:28:29 -0700 Subject: [PATCH 4/5] Replace given null values with Apiary nulls and provide doesNotExists option --- .../java/com/google/gcloud/storage/Blob.java | 42 ++++++++++--------- .../com/google/gcloud/storage/Bucket.java | 22 ++++++---- .../google/gcloud/storage/StorageService.java | 4 ++ 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java index 7ef81bf2b240..31b88981187d 100644 --- a/src/main/java/com/google/gcloud/storage/Blob.java +++ b/src/main/java/com/google/gcloud/storage/Blob.java @@ -16,8 +16,10 @@ package com.google.gcloud.storage; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.api.client.util.Data; import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.api.services.storage.model.StorageObject; @@ -124,22 +126,22 @@ public Builder name(String name) { } public Builder contentType(String contentType) { - this.contentType = contentType; + this.contentType = firstNonNull(contentType, Data.nullOf(String.class)); return this; } public Builder contentDisposition(String contentDisposition) { - this.contentDisposition = contentDisposition; + this.contentDisposition = firstNonNull(contentDisposition, Data.nullOf(String.class)); return this; } public Builder contentLanguage(String contentLanguage) { - this.contentLanguage = contentLanguage; + this.contentLanguage = firstNonNull(contentLanguage, Data.nullOf(String.class)); return this; } public Builder contentEncoding(String contentEncoding) { - this.contentEncoding = contentEncoding; + this.contentEncoding = firstNonNull(contentEncoding, Data.nullOf(String.class)); return this; } @@ -149,7 +151,7 @@ Builder componentCount(Integer componentCount) { } public Builder cacheControl(String cacheControl) { - this.cacheControl = cacheControl; + this.cacheControl = firstNonNull(cacheControl, Data.nullOf(String.class)); return this; } @@ -179,12 +181,12 @@ Builder selfLink(String selfLink) { } public Builder md5(String md5) { - this.md5 = md5; + this.md5 = firstNonNull(md5, Data.nullOf(String.class)); return this; } public Builder crc32c(String crc32c) { - this.crc32c = crc32c; + this.crc32c = firstNonNull(crc32c, Data.nullOf(String.class)); return this; } @@ -263,7 +265,7 @@ public String name() { } public String cacheControl() { - return cacheControl; + return Data.isNull(cacheControl) ? null : cacheControl; } public List acl() { @@ -279,19 +281,19 @@ public Long size() { } public String contentType() { - return contentType; + return Data.isNull(contentType) ? null : contentType; } public String contentEncoding() { - return contentEncoding; + return Data.isNull(contentEncoding) ? null : contentEncoding; } public String contentDisposition() { - return contentDisposition; + return Data.isNull(contentDisposition) ? null : contentDisposition; } public String contentLanguage() { - return contentEncoding; + return Data.isNull(contentLanguage) ? null : contentLanguage; } public Integer componentCount() { @@ -307,11 +309,11 @@ public String selfLink() { } public String md5() { - return md5; + return Data.isNull(md5) ? null : md5; } public String crc32c() { - return crc32c; + return Data.isNull(crc32c) ? null : crc32c; } public String mediaLink() { @@ -367,11 +369,11 @@ public Builder toBuilder() { @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("bucket", bucket) - .add("name", name) - .add("size", size) - .add("content-type", contentType) - .add("metadata", metadata) + .add("bucket", bucket()) + .add("name", name()) + .add("size", size()) + .add("content-type", contentType()) + .add("metadata", metadata()) .toString(); } @@ -397,7 +399,7 @@ public boolean equals(Object obj) { if (!(obj instanceof Blob)) { return false; } - return Objects.equals(toPb(), ((Blob)obj).toPb()); + return Objects.equals(toPb(), ((Blob) obj).toPb()); } StorageObject toPb() { diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java index c36b24c80269..793ae0199d5c 100644 --- a/src/main/java/com/google/gcloud/storage/Bucket.java +++ b/src/main/java/com/google/gcloud/storage/Bucket.java @@ -17,9 +17,11 @@ package com.google.gcloud.storage; import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.Lists.transform; import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.client.util.Data; import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.Bucket.Lifecycle; import com.google.api.services.storage.model.Bucket.Lifecycle.Rule; @@ -67,7 +69,6 @@ public final class Bucket implements Serializable { private final Location location; private final StorageClass storageClass; - static final Function FROM_PB_FUNCTION = new Function() { @Override @@ -248,6 +249,9 @@ public static final class StorageClass implements Serializable { private static final long serialVersionUID = 374002156285326563L; private static final ImmutableMap STRING_TO_OPTION; + private static final StorageClass NULL_VALUE = + new StorageClass(Data.nullOf(String.class)); + private final String value; public enum Option { @@ -299,6 +303,8 @@ public static final class Location implements Serializable { private static final long serialVersionUID = 9073107666838637662L; private static final ImmutableMap STRING_TO_OPTION; + private static final Location NULL_VALUE = new Location(Data.nullOf(String.class)); + private final String value; public enum Option { @@ -392,7 +398,7 @@ Builder selfLink(String selfLink) { } public Builder versioningEnabled(Boolean enable) { - this.versioningEnabled = enable; + this.versioningEnabled = firstNonNull(enable, Data.nullOf(Boolean.class)); return this; } @@ -412,12 +418,12 @@ public Builder deleteRules(Iterable rules) { } public Builder storageClass(StorageClass storageClass) { - this.storageClass = storageClass; + this.storageClass = firstNonNull(storageClass, StorageClass.NULL_VALUE); return this; } public Builder location(Location location) { - this.location = location; + this.location = firstNonNull(location, Location.NULL_VALUE); return this; } @@ -493,7 +499,7 @@ public String selfLink() { } public Boolean versioningEnabled() { - return versioningEnabled; + return Data.isNull(versioningEnabled) ? null : versioningEnabled; } public String indexPage() { @@ -521,11 +527,11 @@ public Long metageneration() { } public Location location() { - return location; + return location == null || Data.isNull(location.value) ? null : location; } public StorageClass storageClass() { - return storageClass; + return storageClass == null || Data.isNull(storageClass.value) ? null : storageClass; } public List cors() { @@ -576,7 +582,7 @@ public boolean equals(Object obj) { @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("name", name) + .add("name", name()) .toString(); } diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 62c62ad8baaa..9e95b0f1281d 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -121,6 +121,10 @@ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) { return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry()); } + public static BlobTargetOption doesNotExists() { + return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0); + } + public static BlobTargetOption generationMatch() { return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH); } From 0299172485f5f06dd36a308ade6ef0cf2b784145 Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 13 May 2015 17:45:56 -0700 Subject: [PATCH 5/5] add apiary references to all example actions --- .../google/gcloud/examples/StorageExample.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index 05f1f9336107..b6f54efa1d93 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -121,6 +121,8 @@ public String params() { * This class demonstrates how to retrieve Bucket or Blob metadata. * If more than one blob is supplied a Batch operation would be used to get all blobs metadata * in a single RPC. + * + * @see Objects: get */ private static class InfoAction extends BlobsAction { @Override @@ -166,6 +168,8 @@ public String params() { * This class demonstrates how to delete a blob. * If more than one blob is supplied a Batch operation would be used to delete all requested * blobs in a single RPC. + * + * @see Objects: delete */ private static class DeleteAction extends BlobsAction { @Override @@ -196,6 +200,8 @@ public void run(StorageService storage, Blob... blobs) { /** * This class demonstrates how to list buckets or a bucket's blobs. + * + * @see Objects: list */ private static class ListAction extends StorageAction { @@ -233,6 +239,8 @@ public String params() { /** * This class demonstrates how to create a new Blob or to update its content. + * + * @see Objects: insert */ private static class UploadAction extends StorageAction> { @Override @@ -286,6 +294,8 @@ public String params() { * This class demonstrates how read a blob's content. * The example will dump the content to a local file if one was given or write * it to stdout otherwise. + * + * @see Objects: get */ private static class DownloadAction extends StorageAction> { @@ -354,7 +364,7 @@ public String params() { /** * This class demonstrates how to use the copy command. * - * @see Object copy + * @see Objects: copy */ private static class CopyAction extends StorageAction { @Override @@ -380,7 +390,7 @@ public String params() { /** * This class demonstrates how to use the compose command. * - * @see Object compose + * @see Objects: compose */ private static class ComposeAction extends StorageAction { @Override @@ -411,7 +421,7 @@ public String params() { /** * This class demonstrates how to update a blob's metadata. * - * @see Object compose + * @see Objects: update */ private static class UpdateMetadata extends StorageAction>> {