From 6e4a2e4e8ad813311a183454437e998074f83f65 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 11 May 2015 16:32:32 -0700 Subject: [PATCH 1/4] complete package-info and make get return null on 404 --- .../gcloud/examples/StorageExample.java | 9 ++-- .../google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../google/gcloud/storage/StorageService.java | 5 +-- .../gcloud/storage/StorageServiceImpl.java | 41 +++++++++++++------ .../google/gcloud/storage/package-info.java | 12 +++++- 5 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java index b545e8323dcd..8b1a79b37212 100644 --- a/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -69,7 +69,7 @@ private static abstract class StorageAction { abstract void run(StorageService storage, T request) throws Exception; - abstract T parse(String... args) throws IllegalArgumentException; + abstract T parse(String... args) throws IllegalArgumentException, IOException; protected String params() { return ""; @@ -115,6 +115,8 @@ public String params() { private static class InfoAction extends BlobsAction { @Override public void run(StorageService storage, Blob... blobs) { + + if (blobs.length == 1) { if (blobs[0].name().isEmpty()) { System.out.println(storage.get(blobs[0].bucket())); @@ -217,13 +219,14 @@ public void run(StorageService storage, Tuple tuple) throws Exceptio } @Override - Tuple parse(String... args) { + Tuple parse(String... args) throws IOException { if (args.length < 2 || args.length > 3) { throw new IllegalArgumentException(); } Path path = Paths.get(args[0]); + String contentType = Files.probeContentType(path); String blob = args.length < 3 ? path.getFileName().toString() : args[2]; - return Tuple.of(path, Blob.of(args[1], blob)); + return Tuple.of(path, Blob.builder(args[1], blob).contentType(contentType).build()); } @Override diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 7ebbb1340d0f..455624f79b65 100644 --- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -1 +1 @@ -/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.json.JsonHttpContent; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.projectId(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new AbstractInputStreamContent(storageObject.getContentType()) { @Override public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(content); } @Override public long getLength() throws IOException { return content.length; } @Override public boolean retrySupported() { return true; } }) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.projectId()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937) downloader.setContentRange(position, (int) position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest, long destOffset, int length, boolean last) throws StorageServiceException { try { GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new ByteArrayContent(null, toWrite, toWriteOffset, length)); long limit = destOffset + length; StringBuilder range = new StringBuilder("bytes "); range.append(destOffset).append('-').append(limit - 1).append('/'); if (last) { range.append(limit); } else { range.append('*'); } httpRequest.getHeaders().setContentRange(range.toString()); int code; String message; IOException exception = null; try { HttpResponse response = httpRequest.execute(); code = response.getStatusCode(); message = response.getStatusMessage(); } catch (HttpResponseException ex) { exception = ex; code = ex.getStatusCode(); message = ex.getStatusMessage(); } if (!last && code != 308 || last && !(code == 200 || code == 201)) { if (exception != null) { throw exception; } GoogleJsonError error = new GoogleJsonError(); error.setCode(code); error.setMessage(message); throw translate(error); } } catch (IOException ex) { throw translate(ex); } } @Override public String open(StorageObject object, Map options) throws StorageServiceException { try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object content = option.get(options); if (content != null) { url.set(option.value(), content.toString()); } } JsonFactory jsonFactory = storage.getJsonFactory(); HttpRequestFactory requestFactory = storage.getRequestFactory(); HttpRequest httpRequest = requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object)); httpRequest.getHeaders().set("X-Upload-Content-Type", MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream")); HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } return response.getHeaders().getLocation(); } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file +/* * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.gcloud.spi; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH; import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS; import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL; import static com.google.gcloud.spi.StorageRpc.Option.PREFIX; import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS; import com.google.api.client.googleapis.batch.json.JsonBatchCallback; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.googleapis.media.MediaHttpDownloader; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.json.JsonHttpContent; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.Storage.Objects.Get; import com.google.api.services.storage.Storage.Objects.Insert; import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.Buckets; import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gcloud.storage.StorageServiceException; import com.google.gcloud.storage.StorageServiceOptions; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; public class DefaultStorageRpc implements StorageRpc { public static final String DEFAULT_PROJECTION = "full"; private final StorageServiceOptions options; private final Storage storage; // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408); public DefaultStorageRpc(StorageServiceOptions options) { HttpTransport transport = options.httpTransportFactory().create(); HttpRequestInitializer initializer = options.httpRequestInitializer(); this.options = options; storage = new Storage.Builder(transport, new JacksonFactory(), initializer) .setApplicationName("gcloud-java") .build(); // Todo: make sure nulls are being used as Data.asNull() } private static StorageServiceException translate(IOException exception) { StorageServiceException translated; if (exception instanceof GoogleJsonResponseException) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = new StorageServiceException(0, exception.getMessage(), false); } translated.initCause(exception); return translated; } private static StorageServiceException translate(GoogleJsonError exception) { boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) || "InternalError".equals(exception.getMessage()); return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable); } @Override public Bucket create(Bucket bucket, Map options) throws StorageServiceException { try { return storage.buckets() .insert(this.options.projectId(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject create(StorageObject storageObject, final byte[] content, Map options) throws StorageServiceException { try { return storage.objects() .insert(storageObject.getBucket(), storageObject, new ByteArrayContent(storageObject.getContentType(), content)) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(Map options) { try { Buckets buckets = storage.buckets() .list(this.options.projectId()) .setProjection(DEFAULT_PROJECTION) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Tuple> list(String bucket, Map options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .execute(); return Tuple.>of( objects.getNextPageToken(), objects.getItems()); } catch (IOException ex) { throw translate(ex); } } @Override public Bucket get(Bucket bucket, Map options) { try { return storage.buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject get(StorageObject object, Map options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Get getRequest(StorageObject object, Map options) throws IOException { return storage.objects() .get(object.getBucket(), object.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public Bucket patch(Bucket bucket, Map options) { try { return storage.buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject patch(StorageObject storageObject, Map options) { try { return patchRequest(storageObject, options).execute(); } catch (IOException ex) { throw translate(ex); } } private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options) throws IOException { return storage.objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public boolean delete(Bucket bucket, Map options) { try { storage.buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } @Override public boolean delete(StorageObject blob, Map options) { try { deleteRequest(blob, options).execute(); return true; } catch (IOException ex) { StorageServiceException serviceException = translate(ex); if (serviceException.code() == 404) { return false; } throw serviceException; } } private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options) throws IOException { return storage.objects() .delete(blob.getBucket(), blob.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationMatch(100L) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); } @Override public StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageServiceException { ComposeRequest request = new ComposeRequest(); if (target.getContentType() == null) { // todo: remove once this is no longer requirement (b/20681287). target.setContentType("application/octet-stream"); } request.setDestination(target); List sourceObjects = new ArrayList<>(); for (StorageObject source : sources) { ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects(); sourceObject.setName(source.getName()); Long generation = source.getGeneration(); if (generation != null) { sourceObject.setGeneration(generation); sourceObject.setObjectPreconditions( new ObjectPreconditions().setIfGenerationMatch(generation)); } sourceObjects.add(sourceObject); } request.setSourceObjects(sourceObjects); try { // todo: missing setProjection (b/20659000) return storage.objects() .compose(target.getBucket(), target.getName(), request) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public StorageObject copy(StorageObject source, Map sourceOptions, StorageObject target, Map targetOptions) throws StorageServiceException { try { return storage .objects() .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target.getContentType() != null ? target : null) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) .execute(); } catch (IOException ex) { throw translate(ex); } } @Override public byte[] load(StorageObject from, Map options) throws StorageServiceException { try { Storage.Objects.Get getRequest = storage.objects() .get(from.getBucket(), from.getName()) .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true); getRequest.executeMediaAndDownloadTo(out); return out.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public BatchResponse batch(BatchRequest request) throws StorageServiceException { com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch(); final Map> deletes = Maps.newConcurrentMap(); final Map> updates = Maps.newConcurrentMap(); final Map> gets = Maps.newConcurrentMap(); try { for (final Tuple> tuple : request.toDelete) { deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(Void ignore, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { deletes.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toUpdate) { patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { updates.put(tuple.x(), Tuple.of(null, translate(e))); } }); } for (final Tuple> tuple : request.toGet) { getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() { @Override public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(storageObject, null)); } @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { gets.put(tuple.x(), Tuple.of(null, translate(e))); } }); } batch.execute(); } catch (IOException ex) { throw translate(ex); } return new BatchResponse(deletes, updates, gets); } @Override public byte[] read(StorageObject from, Map options, long position, int bytes) throws StorageServiceException { try { Get req = storage.objects().get(from.getBucket(), from.getName()); req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options)); MediaHttpDownloader downloader = req.getMediaHttpDownloader(); // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937) downloader.setContentRange(position, (int) position + bytes); downloader.setDirectDownloadEnabled(true); ByteArrayOutputStream output = new ByteArrayOutputStream(); req.executeMediaAndDownloadTo(output); return output.toByteArray(); } catch (IOException ex) { throw translate(ex); } } @Override public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest, long destOffset, int length, boolean last) throws StorageServiceException { try { GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, new ByteArrayContent(null, toWrite, toWriteOffset, length)); long limit = destOffset + length; StringBuilder range = new StringBuilder("bytes "); range.append(destOffset).append('-').append(limit - 1).append('/'); if (last) { range.append(limit); } else { range.append('*'); } httpRequest.getHeaders().setContentRange(range.toString()); int code; String message; IOException exception = null; try { HttpResponse response = httpRequest.execute(); code = response.getStatusCode(); message = response.getStatusMessage(); } catch (HttpResponseException ex) { exception = ex; code = ex.getStatusCode(); message = ex.getStatusMessage(); } if (!last && code != 308 || last && !(code == 200 || code == 201)) { if (exception != null) { throw exception; } GoogleJsonError error = new GoogleJsonError(); error.setCode(code); error.setMessage(message); throw translate(error); } } catch (IOException ex) { throw translate(ex); } } @Override public String open(StorageObject object, Map options) throws StorageServiceException { try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); String host = url.getHost(); String path = "/upload" + url.getRawPath(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object content = option.get(options); if (content != null) { url.set(option.value(), content.toString()); } } JsonFactory jsonFactory = storage.getJsonFactory(); HttpRequestFactory requestFactory = storage.getRequestFactory(); HttpRequest httpRequest = requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object)); httpRequest.getHeaders().set("X-Upload-Content-Type", MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream")); HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) { GoogleJsonError error = new GoogleJsonError(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); } return response.getHeaders().getLocation(); } catch (IOException ex) { throw translate(ex); } } } \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 2294704eac33..ca130162d5cd 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -412,14 +412,14 @@ public static Builder builder() { Blob create(Blob blob, byte[] content, BlobTargetOption... options); /** - * Return the requested bucket. + * Return the requested bucket or {@code null} if not found. * * @throws StorageServiceException upon failure */ Bucket get(String bucket, BucketSourceOption... options); /** - * Return the requested blob. + * Return the requested blob or {@code null} if not found. * * @throws StorageServiceException upon failure */ @@ -516,5 +516,4 @@ public static Builder builder() { * @throws StorageServiceException upon failure */ BlobWriteChannel writer(Blob blob, BlobTargetOption... options); - } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index e577cf2c0aa3..b4f692f7e8cc 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -16,6 +16,7 @@ package com.google.gcloud.storage; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; import static com.google.gcloud.RetryHelper.runWithRetries; import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER; @@ -32,7 +33,6 @@ import com.google.api.services.storage.model.StorageObject; import com.google.common.base.Function; import com.google.common.base.Functions; -import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -76,6 +76,7 @@ public RetryResult beforeEval(Exception exception) { }; private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder() .abortOn(RuntimeException.class).interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build(); + private static final byte[] EMPTY_BYTE_ARRAY = {}; private final StorageRpc storageRpc; private final RetryParams retryParams; @@ -83,7 +84,7 @@ public RetryResult beforeEval(Exception exception) { StorageServiceImpl(StorageServiceOptions options) { super(options); storageRpc = options.storageRpc(); - retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries()); + retryParams = firstNonNull(options.retryParams(), RetryParams.noRetries()); // todo: replace nulls with Value.asNull (per toPb) // todo: configure timeouts - https://developers.google.com/api-client-library/java/google-api-java-client/errors // todo: provide rewrite - https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite @@ -111,7 +112,7 @@ public Blob create(Blob blob, final byte[] content, BlobTargetOption... options) return Blob.fromPb(runWithRetries(new Callable() { @Override public StorageObject call() { - return storageRpc.create(blobPb, content, optionsMap); + return storageRpc.create(blobPb, firstNonNull(content, EMPTY_BYTE_ARRAY), optionsMap); } }, retryParams, EXCEPTION_HANDLER)); } @@ -120,25 +121,41 @@ public StorageObject call() { public Bucket get(String bucket, BucketSourceOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = Bucket.of(bucket).toPb(); final Map optionsMap = optionMap(options); - return Bucket.fromPb(runWithRetries( + com.google.api.services.storage.model.Bucket answer = runWithRetries( new Callable() { @Override public com.google.api.services.storage.model.Bucket call() { - return storageRpc.get(bucketPb, optionsMap); + try { + return storageRpc.get(bucketPb, optionsMap); + } catch (StorageServiceException ex) { + if (ex.code() == 404) { + return null; + } + throw ex; + } } - }, retryParams, EXCEPTION_HANDLER)); + }, retryParams, EXCEPTION_HANDLER); + return answer == null ? null : Bucket.fromPb(answer); } @Override public Blob get(String bucket, String blob, BlobSourceOption... options) { final StorageObject storedObject = Blob.of(bucket, blob).toPb(); final Map optionsMap = optionMap(options); - return Blob.fromPb(runWithRetries(new Callable() { + StorageObject storageObject = runWithRetries(new Callable() { @Override public StorageObject call() { - return storageRpc.get(storedObject, optionsMap); + try { + return storageRpc.get(storedObject, optionsMap); + } catch (StorageServiceException ex) { + if (ex.code() == 404) { + return null; + } + throw ex; + } } - }, retryParams, EXCEPTION_HANDLER)); + }, retryParams, EXCEPTION_HANDLER); + return storageObject == null ? null : Blob.fromPb(storageObject); } @Override @@ -368,7 +385,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE private void initTransients() { storageRpc = serviceOptions.storageRpc(); - retryParams = MoreObjects.firstNonNull(serviceOptions.retryParams(), RetryParams.noRetries()); + retryParams = firstNonNull(serviceOptions.retryParams(), RetryParams.noRetries()); storageObject = blob.toPb(); } @@ -504,7 +521,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE private void initTransients() { storageRpc = options.storageRpc(); - retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries()); + retryParams = firstNonNull(options.retryParams(), RetryParams.noRetries()); storageObject = blob.toPb(); } @@ -600,7 +617,7 @@ private static void addToOptionMap(StorageRpc.Option getOption, StorageRpc.O T value = (T) map.remove(getOption); checkArgument(value != null || defaultValue != null, "Option " + getOption.value() + " is missing a value"); - value = MoreObjects.firstNonNull(value, defaultValue); + value = firstNonNull(value, defaultValue); map.put(putOption, value); } } diff --git a/src/main/java/com/google/gcloud/storage/package-info.java b/src/main/java/com/google/gcloud/storage/package-info.java index e938755271eb..c02b743d0977 100644 --- a/src/main/java/com/google/gcloud/storage/package-info.java +++ b/src/main/java/com/google/gcloud/storage/package-info.java @@ -22,8 +22,16 @@ * StorageServiceOptions options = StorageServiceOptions.builder().projectId("project").build(); * StorageService storage = StorageServiceFactory.instance().get(options); * byte[] content = readContent(); - * Blob blob = storage.create(Blob.of("bucket", "blob_name"), content); - * } + * Blob blob = storage.get("bucket", "blob_name"); + * if (blob == null) { + * storage.create(Blob.of("bucket", "blob_name"), content); + * } else { + * byte[] prevContent = storage.load("bucket", "blob_name"); + * content = mergeContent(prevContent, content); + * WritableByteChannel channel = storage.writer(blob); + * channel.write(ByteBuffer.wrap(content)); + * channel.close(); + * }} * * @see Google Cloud Storage */ From 4481a4a580898c49cc902cabf58f51c7a05afb66 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 11 May 2015 16:45:00 -0700 Subject: [PATCH 2/4] add another shortcut for CopyRequest --- src/main/java/com/google/gcloud/storage/StorageService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index ca130162d5cd..4284f1517f10 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -390,6 +390,10 @@ public static CopyRequest of(String sourceBucket, String sourceBlob, Blob target return builder().source(sourceBucket, sourceBlob).target(target).build(); } + public static CopyRequest of(String sourceBucket, String sourceBlob, String targetBlob) { + return of(sourceBucket, sourceBlob, Blob.of(sourceBucket, targetBlob)); + } + public static Builder builder() { return new Builder(); } From 535b0a17db8e3dd4249c311b847200b991798c19 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 11 May 2015 16:49:12 -0700 Subject: [PATCH 3/4] add another shortcut for ComposeRequest --- src/main/java/com/google/gcloud/storage/StorageService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java index 4284f1517f10..62c62ad8baaa 100644 --- a/src/main/java/com/google/gcloud/storage/StorageService.java +++ b/src/main/java/com/google/gcloud/storage/StorageService.java @@ -306,6 +306,10 @@ public static ComposeRequest of(Iterable sources, Blob target) { return builder().target(target).addSource(sources).build(); } + public static ComposeRequest of(String bucket, Iterable sources, String target) { + return of(sources, Blob.of(bucket, target)); + } + public static Builder builder() { return new Builder(); } From 9761b3c91d96f83a7fc801b076f8daca696f8959 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 11 May 2015 17:12:38 -0700 Subject: [PATCH 4/4] make batch gets return null as well on 404 --- .../google/gcloud/storage/BatchResponse.java | 16 ++++++++++++++-- .../gcloud/storage/StorageServiceImpl.java | 17 ++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java index 0cce1300bcec..fab4d964e1c6 100644 --- a/src/main/java/com/google/gcloud/storage/BatchResponse.java +++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java @@ -36,10 +36,12 @@ public class BatchResponse implements Serializable { public static class Result implements Serializable { private static final long serialVersionUID = -1946539570170529094L; + private static final Result EMPTY = new BatchResponse.Result(null); private final T value; private final StorageServiceException exception; + Result(T value) { this.value = value; this.exception = null; @@ -50,13 +52,15 @@ public static class Result implements Serializable { this.value = null; } - /** * Returns the result. * * @throws StorageServiceException if failed */ public T result() throws StorageServiceException { + if (failed()) { + throw failure(); + } return value; } @@ -76,7 +80,15 @@ public boolean failed() { @Override public String toString() { - return MoreObjects.firstNonNull(value, exception).toString(); + return MoreObjects.toStringHelper(this) + .add("value", value) + .add("exception", exception) + .toString(); + } + + @SuppressWarnings("unchecked") + static Result empty() { + return EMPTY; } } diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java index b4f692f7e8cc..42299f2bc6bd 100644 --- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java +++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java @@ -37,6 +37,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; import com.google.gcloud.BaseService; import com.google.gcloud.ExceptionHandler; import com.google.gcloud.ExceptionHandler.Interceptor; @@ -52,6 +54,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Callable; final class StorageServiceImpl extends BaseService implements StorageService { @@ -325,20 +328,28 @@ public BatchResponse apply(BatchRequest batchRequest) { List> updates = transformBatchResult( toUpdate, response.updates, Blob.FROM_PB_FUNCTION); List> gets = transformBatchResult( - toGet, response.gets, Blob.FROM_PB_FUNCTION); + toGet, response.gets, Blob.FROM_PB_FUNCTION, 404); return new BatchResponse(deletes, updates, gets); } private List> transformBatchResult( Iterable>> request, - Map> results, Function transform) { + Map> results, Function transform, + int... nullOnErrorCodes) { + Set nullOnErrorCodesSet = Sets.newHashSet(Ints.asList(nullOnErrorCodes)); List> response = Lists.newArrayListWithCapacity(results.size()); for (Tuple tuple : request) { Tuple result = results.get(tuple.x()); if (result.x() != null) { response.add(new BatchResponse.Result<>(transform.apply(result.x()))); } else { - response.add(new BatchResponse.Result(result.y())); + StorageServiceException exception = result.y(); + if (nullOnErrorCodesSet.contains(exception.code())) { + //noinspection unchecked + response.add(BatchResponse.Result.empty()); + } else { + response.add(new BatchResponse.Result(result.y())); + } } } return response;