Skip to content

Commit

Permalink
Refactor storage integration tests
Browse files Browse the repository at this point in the history 
- Add option to read keyfile with getResourceAsStream
- Use -1 instead of 42 for (meta)generation
- Add delay-timeout to listing
- Make generateBucketName public and static in RemoteGcsHelper
- Add static deleteBucketRecursively to RemoteGcsHelper
- RemoteGcdHelper.create throws exception if env variables not set
- Add javadoc to RemoteGcdHelper
  • Loading branch information
@mziccard
mziccard committed Oct 6, 2015
1 parent f02bd97 commit bfd4fc4
Show file tree
Hide file tree
Showing 2 changed files with 202 additions and 55 deletions.
67 changes: 35 additions & 32 deletions gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,46 +34,44 @@
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Iterator;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;

public class ITStorageTest {

private static StorageOptions options;
private static Storage storage;
private static RemoteGcsHelper gcsHelper;
private static String bucket;

private static final String bucket = RemoteGcsHelper.generateBucketName();
private static final String CONTENT_TYPE = "text/plain";
private static final byte[] BLOB_BYTE_CONTENT = {0xD, 0xE, 0xA, 0xD};
private static final String BLOB_STRING_CONTENT = "Hello Google Cloud Storage!";

@Rule
public ExpectedException thrown = ExpectedException.none();

@BeforeClass
public static void beforeClass() {
gcsHelper = RemoteGcsHelper.create();
if (gcsHelper != null) {
options = gcsHelper.options();
storage = StorageFactory.instance().get(options);
bucket = gcsHelper.bucket();
try {
gcsHelper = RemoteGcsHelper.create();
storage = StorageFactory.instance().get(gcsHelper.options());
storage.create(BucketInfo.of(bucket));
} catch (RemoteGcsHelper.GcsHelperException e) {
// ignore
}
}

@AfterClass
public static void afterClass() {
public static void afterClass()
throws ExecutionException, TimeoutException, InterruptedException {
if (storage != null) {
for (BlobInfo info : storage.list(bucket)) {
storage.delete(bucket, info.name());
if (!RemoteGcsHelper.deleteBucketRecursively(storage, bucket, 5, TimeUnit.SECONDS)) {
throw new RuntimeException("Bucket deletion timed out. Could not delete non-empty bucket");
}
storage.delete(bucket);
}
}

Expand All @@ -82,11 +80,16 @@ public void beforeMethod() {
org.junit.Assume.assumeNotNull(storage);
}

@Test
public void testListBuckets() {
ListResult<BucketInfo> bucketList = storage.list(Storage.BucketListOption.prefix(bucket));
for (BucketInfo bucketInfo : bucketList) {
assertTrue(bucketInfo.name().startsWith(bucket));
@Test(timeout = 5000)
public void testListBuckets() throws InterruptedException {
Iterator<BucketInfo> bucketIterator =
storage.list(Storage.BucketListOption.prefix(bucket)).iterator();
while (!bucketIterator.hasNext()) {
Thread.sleep(500);
bucketIterator = storage.list(Storage.BucketListOption.prefix(bucket)).iterator();
}
while (bucketIterator.hasNext()) {
assertTrue(bucketIterator.next().name().startsWith(bucket));
}
}

Expand Down Expand Up @@ -137,7 +140,7 @@ public void testCreateBlobFail() {
BlobInfo blob = BlobInfo.of(bucket, blobName);
assertNotNull(storage.create(blob));
try {
storage.create(blob.toBuilder().generation(42L).build(), BLOB_BYTE_CONTENT,
storage.create(blob.toBuilder().generation(-1L).build(), BLOB_BYTE_CONTENT,
Storage.BlobTargetOption.generationMatch());
fail("StorageException was expected");
} catch (StorageException ex) {
Expand Down Expand Up @@ -165,7 +168,7 @@ public void testUpdateBlobFail() {
BlobInfo blob = BlobInfo.of(bucket, blobName);
assertNotNull(storage.create(blob));
try {
storage.update(blob.toBuilder().contentType(CONTENT_TYPE).generation(42L).build(),
storage.update(blob.toBuilder().contentType(CONTENT_TYPE).generation(-1L).build(),
Storage.BlobTargetOption.generationMatch());
fail("StorageException was expected");
} catch (StorageException ex) {
Expand All @@ -187,7 +190,7 @@ public void testDeleteBlobFail() {
BlobInfo blob = BlobInfo.of(bucket, blobName);
assertNotNull(storage.create(blob));
try {
storage.delete(bucket, blob.name(), Storage.BlobSourceOption.generationMatch(42L));
storage.delete(bucket, blob.name(), Storage.BlobSourceOption.generationMatch(-1L));
fail("StorageException was expected");
} catch (StorageException ex) {
// expected
Expand Down Expand Up @@ -232,8 +235,8 @@ public void testComposeBlobFail() {
String targetBlobName = "test-compose-blob-fail-target";
BlobInfo targetBlob = BlobInfo.of(bucket, targetBlobName);
Storage.ComposeRequest req = Storage.ComposeRequest.builder()
.addSource(sourceBlobName1, 42L)
.addSource(sourceBlobName2, 42L)
.addSource(sourceBlobName1, -1L)
.addSource(sourceBlobName2, -1L)
.target(targetBlob)
.build();
try {
Expand Down Expand Up @@ -290,7 +293,7 @@ public void testCopyBlobFail() {
Storage.CopyRequest req = new Storage.CopyRequest.Builder()
.source(bucket, sourceBlobName)
.target(BlobInfo.builder(bucket, targetBlobName).build())
.sourceOptions(Storage.BlobSourceOption.metagenerationMatch(42L))
.sourceOptions(Storage.BlobSourceOption.metagenerationMatch(-1L))
.build();
try {
storage.copy(req);
Expand Down Expand Up @@ -362,11 +365,11 @@ public void testBatchRequestFail() {
String blobName = "test-batch-request-blob-fail";
BlobInfo blob = BlobInfo.of(bucket, blobName);
assertNotNull(storage.create(blob));
BlobInfo updatedBlob = blob.toBuilder().generation(42L).build();
BlobInfo updatedBlob = blob.toBuilder().generation(-1L).build();
BatchRequest batchRequest = BatchRequest.builder()
.update(updatedBlob, Storage.BlobTargetOption.generationMatch())
.delete(bucket, blobName, Storage.BlobSourceOption.generationMatch(42L))
.get(bucket, blobName, Storage.BlobSourceOption.generationMatch(42L))
.delete(bucket, blobName, Storage.BlobSourceOption.generationMatch(-1L))
.get(bucket, blobName, Storage.BlobSourceOption.generationMatch(-1L))
.build();
BatchResponse updateResponse = storage.apply(batchRequest);
assertEquals(1, updateResponse.updates().size());
Expand Down Expand Up @@ -407,7 +410,7 @@ public void testReadChannelFail() throws UnsupportedEncodingException, IOExcepti
BlobInfo blob = BlobInfo.of(bucket, blobName);
assertNotNull(storage.create(blob));
try (BlobReadChannel reader =
storage.reader(bucket, blobName, Storage.BlobSourceOption.metagenerationMatch(42L))) {
storage.reader(bucket, blobName, Storage.BlobSourceOption.metagenerationMatch(-1L))) {
reader.read(ByteBuffer.allocate(42));
fail("StorageException was expected");
} catch (StorageException ex) {
Expand All @@ -419,7 +422,7 @@ public void testReadChannelFail() throws UnsupportedEncodingException, IOExcepti
@Test
public void testWriteChannelFail() throws UnsupportedEncodingException, IOException {
String blobName = "test-write-channel-blob-fail";
BlobInfo blob = BlobInfo.builder(bucket, blobName).generation(42L).build();
BlobInfo blob = BlobInfo.builder(bucket, blobName).generation(-1L).build();
try {
try (BlobWriteChannel writer =
storage.writer(blob, Storage.BlobTargetOption.generationMatch())) {
Expand Down
190 changes: 167 additions & 23 deletions gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,23 @@

package com.google.gcloud.storage;

import com.google.common.collect.ImmutableMap;
import com.google.gcloud.AuthCredentials;
import com.google.gcloud.storage.RemoteGcsHelper.Option.KeyFromClasspath;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.IOException;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;

Expand All @@ -38,53 +47,188 @@ public class RemoteGcsHelper {
private static final String PRIVATE_KEY_ENV_VAR = "GCLOUD_TESTS_KEY";

private final StorageOptions options;
private final String bucket;

private RemoteGcsHelper(StorageOptions options, String bucket) {
private RemoteGcsHelper(StorageOptions options) {
this.options = options;
this.bucket = bucket;
}

/**
* Returns a {@StorageOptions} object to be used for testing.
*/
public StorageOptions options() {
return options;
}

public String bucket() {
return bucket;
/**
* Delete a bucket recursively. Objects in the bucket are listed and deleted until bucket deletion
* succeeds or {@code timeout} expires.
*
* @param storage the storage service to be used to issue requests
* @param bucket the bucket to be deleted
* @param timeout the maximum time to wait
* @param unit the time unit of the timeout argument
* @return true if deletion succeeded, false if timeout expired.
* @throws InterruptedException if the thread deleting the bucket is interrupted while waiting
* @throws ExecutionException if an exception was thrown while deleting bucket or bucket objects
*/
public static Boolean deleteBucketRecursively(Storage storage, String bucket, long timeout,
TimeUnit unit) throws InterruptedException, ExecutionException {
ExecutorService executor = Executors.newSingleThreadExecutor();
Future<Boolean> future = executor.submit(new DeleteBucketTask(storage, bucket));
try {
return future.get(timeout, unit);
} catch (TimeoutException ex) {
return false;
}
}

private static String generateBucketName() {

/**
* Returns a bucket name generated using a random UUID.
*/
public static String generateBucketName() {
return BUCKET_NAME_PREFIX + UUID.randomUUID().toString();
}

public static RemoteGcsHelper create() {
if (System.getenv(PROJECT_ID_ENV_VAR) == null || System.getenv(PRIVATE_KEY_ENV_VAR) == null) {
if (log.isLoggable(Level.WARNING)) {
log.log(Level.INFO, "Environment variables {0} and {1} not set", new String[] {
PROJECT_ID_ENV_VAR, PRIVATE_KEY_ENV_VAR});
}
return null;
}
/**
* Creates a {@code RemoteGcsHelper} object.
*
* @param options creation options
* @return A {@code RemoteGcsHelper} object for the provided options.
* @throws com.google.gcloud.storage.RemoteGcsHelper.GcsHelperException if environment variables
* {@code GCLOUD_TESTS_PROJECT_ID} and {@code GCLOUD_TESTS_KEY_PATH} are not set or if the file
* pointed by {@code GCLOUD_TESTS_KEY_PATH} does not exist
*/
public static RemoteGcsHelper create(Option... options) throws GcsHelperException {
boolean keyFromClassPath = false;
Map<Class<? extends Option>, Option> optionsMap = Option.asImmutableMap(options);
if (optionsMap.containsKey(KeyFromClasspath.class)) {
keyFromClassPath =
((KeyFromClasspath) optionsMap.get(KeyFromClasspath.class)).keyFromClasspath();
}
String projectId = System.getenv(PROJECT_ID_ENV_VAR);
String stringKeyPath = System.getenv(PRIVATE_KEY_ENV_VAR);
File keyFile = new File(stringKeyPath);
if (projectId == null) {
String message = "Environment variable " + PROJECT_ID_ENV_VAR + " not set";
if (log.isLoggable(Level.WARNING)) {
log.log(Level.WARNING, message);
}
throw new GcsHelperException(message);
}
if (stringKeyPath == null) {
String message = "Environment variable " + PRIVATE_KEY_ENV_VAR + " not set";
if (log.isLoggable(Level.WARNING)) {
log.log(Level.WARNING, message);
}
throw new GcsHelperException(message);
}
try {
InputStream keyFileStream = new FileInputStream(keyFile);
StorageOptions options = StorageOptions.builder()
InputStream keyFileStream;
if (keyFromClassPath) {
keyFileStream = RemoteGcsHelper.class.getResourceAsStream(stringKeyPath);
if (keyFileStream == null) {
throw new FileNotFoundException(stringKeyPath + " not found in classpath");
}
} else {
keyFileStream = new FileInputStream(stringKeyPath);
}
StorageOptions storageOptions = StorageOptions.builder()
.authCredentials(AuthCredentials.createForJson(keyFileStream))
.projectId(projectId)
.build();
return new RemoteGcsHelper(options, generateBucketName());
return new RemoteGcsHelper(storageOptions);
} catch (FileNotFoundException ex) {
if (log.isLoggable(Level.WARNING)) {
log.log(Level.WARNING, ex.getMessage());
}
return null;
throw GcsHelperException.translate(ex);
} catch (IOException ex) {
if (log.isLoggable(Level.WARNING)) {
log.log(Level.WARNING, ex.getMessage());
}
return null;
throw GcsHelperException.translate(ex);
}
}

private static class DeleteBucketTask implements Callable<Boolean> {

private Storage storage;
private String bucket;

public DeleteBucketTask(Storage storage, String bucket) {
this.storage = storage;
this.bucket = bucket;
}

@Override
public Boolean call() throws Exception {
while (true) {
for (BlobInfo info : storage.list(bucket)) {
storage.delete(bucket, info.name());
}
try {
storage.delete(bucket);
return true;
} catch (StorageException e) {
if (e.code() == 409) {
Thread.sleep(500);
} else {
throw e;
}
}
}
}
}

public static abstract class Option implements java.io.Serializable {

private static final long serialVersionUID = 8849118657896662369L;

public static final class KeyFromClasspath extends Option {

private static final long serialVersionUID = -5506049413185246821L;

private final boolean keyFromClasspath;

public KeyFromClasspath(boolean keyFromClasspath) {
this.keyFromClasspath = keyFromClasspath;
}

public boolean keyFromClasspath() {
return keyFromClasspath;
}
}

Option() {
// package protected
}

public static KeyFromClasspath keyFromClassPath() {
return new KeyFromClasspath(true);
}

static Map<Class<? extends Option>, Option> asImmutableMap(Option... options) {
ImmutableMap.Builder<Class<? extends Option>, Option> builder = ImmutableMap.builder();
for (Option option : options) {
builder.put(option.getClass(), option);
}
return builder.build();
}
}

public static class GcsHelperException extends RuntimeException {

private static final long serialVersionUID = -7756074894502258736L;

public GcsHelperException(String message) {
super(message);
}

public GcsHelperException(String message, Throwable cause) {
super(message, cause);
}

public static GcsHelperException translate(Exception ex) {
return new GcsHelperException(ex.getMessage(), ex);
}
}
}

0 comments on commit bfd4fc4

Please sign in to comment.