Skip to content

Commit

Permalink
Remove md5 and crc32c options from StorageRpc
Browse files Browse the repository at this point in the history 
- BlobWriteOption extends Serializable instead of Option
- add BlobTargetObject.convert to convert BlobWriteOption and possibly mask md5 and crc32c fields
- add private Storage.create(blobInfo, inputStream, blobTargetOption)
  • Loading branch information
@mziccard
mziccard committed Oct 20, 2015
1 parent cebb0bb commit 4f24df3
Show file tree
Hide file tree
Showing 5 changed files with 96 additions and 65 deletions.
13 changes: 0 additions & 13 deletions gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_MD5_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_CRC32C_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
Expand Down Expand Up @@ -108,15 +106,6 @@ private static StorageException translate(GoogleJsonError exception) {
return new StorageException(exception.getCode(), exception.getMessage(), retryable);
}

private static void applyOptions(StorageObject storageObject, Map<Option, ?> options) {
if (IF_MD5_MATCH.getBoolean(options) == null) {
storageObject.setMd5Hash(null);
}
if (IF_CRC32C_MATCH.getBoolean(options) == null) {
storageObject.setCrc32c(null);
}
}

@Override
public Bucket create(Bucket bucket, Map<Option, ?> options) throws StorageException {
try {
Expand All @@ -134,7 +123,6 @@ public Bucket create(Bucket bucket, Map<Option, ?> options) throws StorageExcept
@Override
public StorageObject create(StorageObject storageObject, final InputStream content,
Map<Option, ?> options) throws StorageException {
applyOptions(storageObject, options);
try {
Storage.Objects.Insert insert = storage.objects()
.insert(storageObject.getBucket(), storageObject,
Expand Down Expand Up @@ -503,7 +491,6 @@ public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObj
@Override
public String open(StorageObject object, Map<Option, ?> options)
throws StorageException {
applyOptions(object, options);
try {
Insert req = storage.objects().insert(object.getBucket(), object);
GenericUrl url = req.buildHttpRequest().getUrl();
Expand Down
2 changes: 0 additions & 2 deletions gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ enum Option {
IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
IF_MD5_MATCH("md5Hash"),
IF_CRC32C_MATCH("crc32c"),
PREFIX("prefix"),
MAX_RESULTS("maxResults"),
PAGE_TOKEN("pageToken"),
Expand Down
89 changes: 65 additions & 24 deletions gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
import com.google.gcloud.Service;
import com.google.gcloud.spi.StorageRpc;
import com.google.gcloud.spi.StorageRpc.Tuple;

import java.io.InputStream;
import java.io.Serializable;
Expand All @@ -33,6 +35,7 @@
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;

Expand Down Expand Up @@ -146,65 +149,103 @@ public static BlobTargetOption metagenerationNotMatch() {
return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH);
}

static BlobWriteOption[] convert(BlobTargetOption[] options, BlobWriteOption... optionsToAdd) {
BlobWriteOption[] writeOptions = new BlobWriteOption[options.length + optionsToAdd.length];
int index = 0;
for (BlobTargetOption option : options) {
writeOptions[index++] = new BlobWriteOption(option);
}
for (BlobWriteOption option : optionsToAdd) {
writeOptions[index++] = option;
static Tuple<BlobInfo, BlobTargetOption[]> convert(BlobInfo info, BlobWriteOption... options) {
BlobInfo.Builder infoBuilder = info.toBuilder().crc32c(null).md5(null);
List<BlobTargetOption> targetOptions = Lists.newArrayListWithCapacity(options.length);
for (BlobWriteOption option : options) {
switch (option.option) {
case IF_CRC32C_MATCH:
infoBuilder.crc32c(info.crc32c());
break;
case IF_MD5_MATCH:
infoBuilder.md5(info.md5());
break;
default:
targetOptions.add(option.toTargetOption());
break;
}
}
return writeOptions;
return Tuple.of(infoBuilder.build(),
targetOptions.toArray(new BlobTargetOption[targetOptions.size()]));
}
}

class BlobWriteOption extends Option {
class BlobWriteOption implements Serializable {

private static final long serialVersionUID = -3880421670966224580L;

BlobWriteOption(BlobTargetOption option) {
super(option.rpcOption(), option.value());
private final Option option;
private final Object value;

enum Option {
PREDEFINED_ACL, IF_GENERATION_MATCH, IF_GENERATION_NOT_MATCH, IF_METAGENERATION_MATCH,
IF_METAGENERATION_NOT_MATCH, IF_MD5_MATCH, IF_CRC32C_MATCH;

StorageRpc.Option toRpcOption() {
return StorageRpc.Option.valueOf(this.name());
}
}

private BlobWriteOption(StorageRpc.Option rpcOption, Object value) {
super(rpcOption, value);
BlobTargetOption toTargetOption() {
return new BlobTargetOption(this.option.toRpcOption(), this.value);
}

private BlobWriteOption(StorageRpc.Option rpcOption) {
this(rpcOption, null);
private BlobWriteOption(Option option, Object value) {
this.option = option;
this.value = value;
}

private BlobWriteOption(Option option) {
this(option, null);
}

@Override
public int hashCode() {
return Objects.hash(option, value);
}

@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (!(obj instanceof BlobWriteOption)) {
return false;
}
final BlobWriteOption other = (BlobWriteOption) obj;
return this.option == other.option && Objects.equals(this.value, other.value);
}

public static BlobWriteOption predefinedAcl(PredefinedAcl acl) {
return new BlobWriteOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
return new BlobWriteOption(Option.PREDEFINED_ACL, acl.entry());
}

public static BlobWriteOption doesNotExist() {
return new BlobWriteOption(StorageRpc.Option.IF_GENERATION_MATCH, 0L);
return new BlobWriteOption(Option.IF_GENERATION_MATCH, 0L);
}

public static BlobWriteOption generationMatch() {
return new BlobWriteOption(StorageRpc.Option.IF_GENERATION_MATCH);
return new BlobWriteOption(Option.IF_GENERATION_MATCH);
}

public static BlobWriteOption generationNotMatch() {
return new BlobWriteOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH);
return new BlobWriteOption(Option.IF_GENERATION_NOT_MATCH);
}

public static BlobWriteOption metagenerationMatch() {
return new BlobWriteOption(StorageRpc.Option.IF_METAGENERATION_MATCH);
return new BlobWriteOption(Option.IF_METAGENERATION_MATCH);
}

public static BlobWriteOption metagenerationNotMatch() {
return new BlobWriteOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH);
return new BlobWriteOption(Option.IF_METAGENERATION_NOT_MATCH);
}

public static BlobWriteOption md5Match() {
return new BlobWriteOption(StorageRpc.Option.IF_MD5_MATCH, true);
return new BlobWriteOption(Option.IF_MD5_MATCH, true);
}

public static BlobWriteOption crc32cMatch() {
return new BlobWriteOption(StorageRpc.Option.IF_CRC32C_MATCH, true);
return new BlobWriteOption(Option.IF_CRC32C_MATCH, true);
}
}

Expand Down
21 changes: 14 additions & 7 deletions gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,7 @@ public BlobInfo create(BlobInfo blobInfo, BlobTargetOption... options) {
.md5(EMPTY_BYTE_ARRAY_MD5)
.crc32c(EMPTY_BYTE_ARRAY_CRC32C)
.build();
return create(updatedInfo, new ByteArrayInputStream(EMPTY_BYTE_ARRAY),
BlobTargetOption.convert(
options, BlobWriteOption.md5Match(), BlobWriteOption.crc32cMatch()));
return create(updatedInfo, new ByteArrayInputStream(EMPTY_BYTE_ARRAY), options);
}

@Override
Expand All @@ -142,14 +140,18 @@ public BlobInfo create(BlobInfo blobInfo, byte[] content, BlobTargetOption... op
.crc32c(BaseEncoding.base64().encode(
Ints.toByteArray(Hashing.crc32c().hashBytes(content).asInt())))
.build();
return create(updatedInfo, new ByteArrayInputStream(content), BlobTargetOption.convert(
options, BlobWriteOption.md5Match(), BlobWriteOption.crc32cMatch()));
return create(updatedInfo, new ByteArrayInputStream(content), options);
}

@Override
public BlobInfo create(BlobInfo blobInfo, final InputStream content, BlobWriteOption... options) {
final StorageObject blobPb = blobInfo.toPb();
final Map<StorageRpc.Option, ?> optionsMap = optionMap(blobInfo, options);
Tuple<BlobInfo, BlobTargetOption[]> targetOptions = BlobTargetOption.convert(blobInfo, options);
return create(targetOptions.x(), content, targetOptions.y());
}

private BlobInfo create(BlobInfo info, final InputStream content, BlobTargetOption... options) {
final StorageObject blobPb = info.toPb();
final Map<StorageRpc.Option, ?> optionsMap = optionMap(info, options);
try {
return BlobInfo.fromPb(runWithRetries(new Callable<StorageObject>() {
@Override
Expand Down Expand Up @@ -558,6 +560,11 @@ public BlobReadChannel reader(BlobId blob, BlobSourceOption... options) {

@Override
public BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) {
Tuple<BlobInfo, BlobTargetOption[]> targetOptions = BlobTargetOption.convert(blobInfo, options);
return writer(targetOptions.x(), targetOptions.y());
}

private BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options) {
final Map<StorageRpc.Option, ?> optionsMap = optionMap(blobInfo, options);
return new BlobWriteChannelImpl(options(), blobInfo, optionsMap);
}
Expand Down
36 changes: 17 additions & 19 deletions gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,15 +134,6 @@ public class StorageImplTest {
Storage.BlobWriteOption.md5Match();
private static final Storage.BlobWriteOption BLOB_WRITE_CRC2C =
Storage.BlobWriteOption.crc32cMatch();
private static final Map<StorageRpc.Option, ?> BLOB_WRITE_OPTIONS_SIMPLE = ImmutableMap.of(
StorageRpc.Option.IF_MD5_MATCH, true,
StorageRpc.Option.IF_CRC32C_MATCH, true);
private static final Map<StorageRpc.Option, ?> BLOB_WRITE_OPTIONS_COMPLEX = ImmutableMap.of(
StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_INFO1.metageneration(),
StorageRpc.Option.IF_GENERATION_MATCH, 0L,
StorageRpc.Option.PREDEFINED_ACL, BUCKET_TARGET_PREDEFINED_ACL.value(),
StorageRpc.Option.IF_MD5_MATCH, true,
StorageRpc.Option.IF_CRC32C_MATCH, true);

// Bucket source options
private static final Storage.BucketSourceOption BUCKET_SOURCE_METAGENERATION =
Expand Down Expand Up @@ -276,7 +267,7 @@ public void testCreateBlob() throws IOException {
EasyMock.expect(storageRpcMock.create(
EasyMock.eq(BLOB_INFO1.toBuilder().md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build().toPb()),
EasyMock.capture(capturedStream),
EasyMock.eq(BLOB_WRITE_OPTIONS_SIMPLE)))
EasyMock.eq(EMPTY_RPC_OPTIONS)))
.andReturn(BLOB_INFO1.toPb());
EasyMock.replay(optionsMock, storageRpcMock);
storage = StorageFactory.instance().get(optionsMock);
Expand All @@ -301,7 +292,7 @@ public void testCreateEmptyBlob() throws IOException {
.build()
.toPb()),
EasyMock.capture(capturedStream),
EasyMock.eq(BLOB_WRITE_OPTIONS_SIMPLE)))
EasyMock.eq(EMPTY_RPC_OPTIONS)))
.andReturn(BLOB_INFO1.toPb());
EasyMock.replay(optionsMock, storageRpcMock);
storage = StorageFactory.instance().get(optionsMock);
Expand All @@ -324,7 +315,7 @@ public void testCreateBlobWithOptions() throws IOException {
.build()
.toPb()),
EasyMock.capture(capturedStream),
EasyMock.eq(BLOB_WRITE_OPTIONS_COMPLEX)))
EasyMock.eq(BLOB_TARGET_OPTIONS_CREATE)))
.andReturn(BLOB_INFO1.toPb());
EasyMock.replay(optionsMock, storageRpcMock);
storage = StorageFactory.instance().get(optionsMock);
Expand All @@ -344,11 +335,14 @@ public void testCreateBlobFromStream() throws IOException {
EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock);
EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries());
ByteArrayInputStream fileStream = new ByteArrayInputStream(BLOB_CONTENT);
EasyMock.expect(storageRpcMock.create(BLOB_INFO1.toPb(), fileStream, EMPTY_RPC_OPTIONS))
BlobInfo.Builder infoBuilder = BLOB_INFO1.toBuilder();
BlobInfo infoWithHashes = infoBuilder.md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build();
BlobInfo infoWithoutHashes = infoBuilder.md5(null).crc32c(null).build();
EasyMock.expect(storageRpcMock.create(infoWithoutHashes.toPb(), fileStream, EMPTY_RPC_OPTIONS))
.andReturn(BLOB_INFO1.toPb());
EasyMock.replay(optionsMock, storageRpcMock);
storage = StorageFactory.instance().get(optionsMock);
BlobInfo blob = storage.create(BLOB_INFO1, fileStream);
BlobInfo blob = storage.create(infoWithHashes, fileStream);
assertEquals(BLOB_INFO1, blob);
}

Expand Down Expand Up @@ -807,24 +801,28 @@ public void testReaderWithOptions() throws IOException {
@Test
public void testWriter() {
EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock).times(2);
EasyMock.expect(storageRpcMock.open(BLOB_INFO1.toPb(), EMPTY_RPC_OPTIONS))
BlobInfo.Builder infoBuilder = BLOB_INFO1.toBuilder();
BlobInfo infoWithHashes = infoBuilder.md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build();
BlobInfo infoWithoutHashes = infoBuilder.md5(null).crc32c(null).build();
EasyMock.expect(storageRpcMock.open(infoWithoutHashes.toPb(), EMPTY_RPC_OPTIONS))
.andReturn("upload-id");
EasyMock.replay(optionsMock, storageRpcMock);
storage = StorageFactory.instance().get(optionsMock);
BlobWriteChannel channel = storage.writer(BLOB_INFO1);
BlobWriteChannel channel = storage.writer(infoWithHashes);
assertNotNull(channel);
assertTrue(channel.isOpen());
}

@Test
public void testWriterWithOptions() {
EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock).times(2);
EasyMock.expect(storageRpcMock.open(BLOB_INFO1.toPb(), BLOB_WRITE_OPTIONS_COMPLEX))
BlobInfo info = BLOB_INFO1.toBuilder().md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build();
EasyMock.expect(storageRpcMock.open(info.toPb(), BLOB_TARGET_OPTIONS_CREATE))
.andReturn("upload-id");
EasyMock.replay(optionsMock, storageRpcMock);
storage = StorageFactory.instance().get(optionsMock);
BlobWriteChannel channel = storage.writer(BLOB_INFO1, BLOB_WRITE_METAGENERATION,
BLOB_WRITE_NOT_EXIST, BLOB_WRITE_PREDEFINED_ACL, BLOB_WRITE_CRC2C, BLOB_WRITE_MD5_HASH);
BlobWriteChannel channel = storage.writer(info, BLOB_WRITE_METAGENERATION, BLOB_WRITE_NOT_EXIST,
BLOB_WRITE_PREDEFINED_ACL, BLOB_WRITE_CRC2C, BLOB_WRITE_MD5_HASH);
assertNotNull(channel);
assertTrue(channel.isOpen());
}
Expand Down

0 comments on commit 4f24df3

Please sign in to comment.