diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index 714c680601f2..74833bd8fe3d 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -34,46 +34,44 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Calendar; +import java.util.Iterator; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; public class ITStorageTest { - private static StorageOptions options; private static Storage storage; private static RemoteGcsHelper gcsHelper; - private static String bucket; + private static final String bucket = RemoteGcsHelper.generateBucketName(); private static final String CONTENT_TYPE = "text/plain"; private static final byte[] BLOB_BYTE_CONTENT = {0xD, 0xE, 0xA, 0xD}; private static final String BLOB_STRING_CONTENT = "Hello Google Cloud Storage!"; - @Rule - public ExpectedException thrown = ExpectedException.none(); - @BeforeClass public static void beforeClass() { - gcsHelper = RemoteGcsHelper.create(); - if (gcsHelper != null) { - options = gcsHelper.options(); - storage = StorageFactory.instance().get(options); - bucket = gcsHelper.bucket(); + try { + gcsHelper = RemoteGcsHelper.create(); + storage = StorageFactory.instance().get(gcsHelper.options()); storage.create(BucketInfo.of(bucket)); + } catch (RemoteGcsHelper.GcsHelperException e) { + // ignore } } @AfterClass - public static void afterClass() { + public static void afterClass() + throws ExecutionException, TimeoutException, InterruptedException { if (storage != null) { - for (BlobInfo info : storage.list(bucket)) { - storage.delete(bucket, info.name()); + if (!RemoteGcsHelper.deleteBucketRecursively(storage, bucket, 5, TimeUnit.SECONDS)) { + throw new RuntimeException("Bucket deletion timed out. Could not delete non-empty bucket"); } - storage.delete(bucket); } } @@ -82,11 +80,16 @@ public void beforeMethod() { org.junit.Assume.assumeNotNull(storage); } - @Test - public void testListBuckets() { - ListResult bucketList = storage.list(Storage.BucketListOption.prefix(bucket)); - for (BucketInfo bucketInfo : bucketList) { - assertTrue(bucketInfo.name().startsWith(bucket)); + @Test(timeout = 5000) + public void testListBuckets() throws InterruptedException { + Iterator bucketIterator = + storage.list(Storage.BucketListOption.prefix(bucket)).iterator(); + while (!bucketIterator.hasNext()) { + Thread.sleep(500); + bucketIterator = storage.list(Storage.BucketListOption.prefix(bucket)).iterator(); + } + while (bucketIterator.hasNext()) { + assertTrue(bucketIterator.next().name().startsWith(bucket)); } } @@ -137,7 +140,7 @@ public void testCreateBlobFail() { BlobInfo blob = BlobInfo.of(bucket, blobName); assertNotNull(storage.create(blob)); try { - storage.create(blob.toBuilder().generation(42L).build(), BLOB_BYTE_CONTENT, + storage.create(blob.toBuilder().generation(-1L).build(), BLOB_BYTE_CONTENT, Storage.BlobTargetOption.generationMatch()); fail("StorageException was expected"); } catch (StorageException ex) { @@ -165,7 +168,7 @@ public void testUpdateBlobFail() { BlobInfo blob = BlobInfo.of(bucket, blobName); assertNotNull(storage.create(blob)); try { - storage.update(blob.toBuilder().contentType(CONTENT_TYPE).generation(42L).build(), + storage.update(blob.toBuilder().contentType(CONTENT_TYPE).generation(-1L).build(), Storage.BlobTargetOption.generationMatch()); fail("StorageException was expected"); } catch (StorageException ex) { @@ -187,7 +190,7 @@ public void testDeleteBlobFail() { BlobInfo blob = BlobInfo.of(bucket, blobName); assertNotNull(storage.create(blob)); try { - storage.delete(bucket, blob.name(), Storage.BlobSourceOption.generationMatch(42L)); + storage.delete(bucket, blob.name(), Storage.BlobSourceOption.generationMatch(-1L)); fail("StorageException was expected"); } catch (StorageException ex) { // expected @@ -232,8 +235,8 @@ public void testComposeBlobFail() { String targetBlobName = "test-compose-blob-fail-target"; BlobInfo targetBlob = BlobInfo.of(bucket, targetBlobName); Storage.ComposeRequest req = Storage.ComposeRequest.builder() - .addSource(sourceBlobName1, 42L) - .addSource(sourceBlobName2, 42L) + .addSource(sourceBlobName1, -1L) + .addSource(sourceBlobName2, -1L) .target(targetBlob) .build(); try { @@ -290,7 +293,7 @@ public void testCopyBlobFail() { Storage.CopyRequest req = new Storage.CopyRequest.Builder() .source(bucket, sourceBlobName) .target(BlobInfo.builder(bucket, targetBlobName).build()) - .sourceOptions(Storage.BlobSourceOption.metagenerationMatch(42L)) + .sourceOptions(Storage.BlobSourceOption.metagenerationMatch(-1L)) .build(); try { storage.copy(req); @@ -362,11 +365,11 @@ public void testBatchRequestFail() { String blobName = "test-batch-request-blob-fail"; BlobInfo blob = BlobInfo.of(bucket, blobName); assertNotNull(storage.create(blob)); - BlobInfo updatedBlob = blob.toBuilder().generation(42L).build(); + BlobInfo updatedBlob = blob.toBuilder().generation(-1L).build(); BatchRequest batchRequest = BatchRequest.builder() .update(updatedBlob, Storage.BlobTargetOption.generationMatch()) - .delete(bucket, blobName, Storage.BlobSourceOption.generationMatch(42L)) - .get(bucket, blobName, Storage.BlobSourceOption.generationMatch(42L)) + .delete(bucket, blobName, Storage.BlobSourceOption.generationMatch(-1L)) + .get(bucket, blobName, Storage.BlobSourceOption.generationMatch(-1L)) .build(); BatchResponse updateResponse = storage.apply(batchRequest); assertEquals(1, updateResponse.updates().size()); @@ -407,7 +410,7 @@ public void testReadChannelFail() throws UnsupportedEncodingException, IOExcepti BlobInfo blob = BlobInfo.of(bucket, blobName); assertNotNull(storage.create(blob)); try (BlobReadChannel reader = - storage.reader(bucket, blobName, Storage.BlobSourceOption.metagenerationMatch(42L))) { + storage.reader(bucket, blobName, Storage.BlobSourceOption.metagenerationMatch(-1L))) { reader.read(ByteBuffer.allocate(42)); fail("StorageException was expected"); } catch (StorageException ex) { @@ -419,7 +422,7 @@ public void testReadChannelFail() throws UnsupportedEncodingException, IOExcepti @Test public void testWriteChannelFail() throws UnsupportedEncodingException, IOException { String blobName = "test-write-channel-blob-fail"; - BlobInfo blob = BlobInfo.builder(bucket, blobName).generation(42L).build(); + BlobInfo blob = BlobInfo.builder(bucket, blobName).generation(-1L).build(); try { try (BlobWriteChannel writer = storage.writer(blob, Storage.BlobTargetOption.generationMatch())) { diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelper.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelper.java index 8a6ee21ce931..2c79aaa78afe 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelper.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelper.java @@ -16,14 +16,23 @@ package com.google.gcloud.storage; +import com.google.common.collect.ImmutableMap; import com.google.gcloud.AuthCredentials; +import com.google.gcloud.storage.RemoteGcsHelper.Option.KeyFromClasspath; -import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.IOException; import java.io.InputStream; +import java.io.IOException; +import java.util.Map; import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.logging.Level; import java.util.logging.Logger; @@ -38,53 +47,188 @@ public class RemoteGcsHelper { private static final String PRIVATE_KEY_ENV_VAR = "GCLOUD_TESTS_KEY"; private final StorageOptions options; - private final String bucket; - private RemoteGcsHelper(StorageOptions options, String bucket) { + private RemoteGcsHelper(StorageOptions options) { this.options = options; - this.bucket = bucket; } + /** + * Returns a {@StorageOptions} object to be used for testing. + */ public StorageOptions options() { return options; } - public String bucket() { - return bucket; + /** + * Delete a bucket recursively. Objects in the bucket are listed and deleted until bucket deletion + * succeeds or {@code timeout} expires. + * + * @param storage the storage service to be used to issue requests + * @param bucket the bucket to be deleted + * @param timeout the maximum time to wait + * @param unit the time unit of the timeout argument + * @return true if deletion succeeded, false if timeout expired. + * @throws InterruptedException if the thread deleting the bucket is interrupted while waiting + * @throws ExecutionException if an exception was thrown while deleting bucket or bucket objects + */ + public static Boolean deleteBucketRecursively(Storage storage, String bucket, long timeout, + TimeUnit unit) throws InterruptedException, ExecutionException { + ExecutorService executor = Executors.newSingleThreadExecutor(); + Future future = executor.submit(new DeleteBucketTask(storage, bucket)); + try { + return future.get(timeout, unit); + } catch (TimeoutException ex) { + return false; + } } - - private static String generateBucketName() { + + /** + * Returns a bucket name generated using a random UUID. + */ + public static String generateBucketName() { return BUCKET_NAME_PREFIX + UUID.randomUUID().toString(); } - public static RemoteGcsHelper create() { - if (System.getenv(PROJECT_ID_ENV_VAR) == null || System.getenv(PRIVATE_KEY_ENV_VAR) == null) { - if (log.isLoggable(Level.WARNING)) { - log.log(Level.INFO, "Environment variables {0} and {1} not set", new String[] { - PROJECT_ID_ENV_VAR, PRIVATE_KEY_ENV_VAR}); - } - return null; - } + /** + * Creates a {@code RemoteGcsHelper} object. + * + * @param options creation options + * @return A {@code RemoteGcsHelper} object for the provided options. + * @throws com.google.gcloud.storage.RemoteGcsHelper.GcsHelperException if environment variables + * {@code GCLOUD_TESTS_PROJECT_ID} and {@code GCLOUD_TESTS_KEY_PATH} are not set or if the file + * pointed by {@code GCLOUD_TESTS_KEY_PATH} does not exist + */ + public static RemoteGcsHelper create(Option... options) throws GcsHelperException { + boolean keyFromClassPath = false; + Map, Option> optionsMap = Option.asImmutableMap(options); + if (optionsMap.containsKey(KeyFromClasspath.class)) { + keyFromClassPath = + ((KeyFromClasspath) optionsMap.get(KeyFromClasspath.class)).keyFromClasspath(); + } String projectId = System.getenv(PROJECT_ID_ENV_VAR); String stringKeyPath = System.getenv(PRIVATE_KEY_ENV_VAR); - File keyFile = new File(stringKeyPath); + if (projectId == null) { + String message = "Environment variable " + PROJECT_ID_ENV_VAR + " not set"; + if (log.isLoggable(Level.WARNING)) { + log.log(Level.WARNING, message); + } + throw new GcsHelperException(message); + } + if (stringKeyPath == null) { + String message = "Environment variable " + PRIVATE_KEY_ENV_VAR + " not set"; + if (log.isLoggable(Level.WARNING)) { + log.log(Level.WARNING, message); + } + throw new GcsHelperException(message); + } try { - InputStream keyFileStream = new FileInputStream(keyFile); - StorageOptions options = StorageOptions.builder() + InputStream keyFileStream; + if (keyFromClassPath) { + keyFileStream = RemoteGcsHelper.class.getResourceAsStream(stringKeyPath); + if (keyFileStream == null) { + throw new FileNotFoundException(stringKeyPath + " not found in classpath"); + } + } else { + keyFileStream = new FileInputStream(stringKeyPath); + } + StorageOptions storageOptions = StorageOptions.builder() .authCredentials(AuthCredentials.createForJson(keyFileStream)) .projectId(projectId) .build(); - return new RemoteGcsHelper(options, generateBucketName()); + return new RemoteGcsHelper(storageOptions); } catch (FileNotFoundException ex) { if (log.isLoggable(Level.WARNING)) { log.log(Level.WARNING, ex.getMessage()); } - return null; + throw GcsHelperException.translate(ex); } catch (IOException ex) { if (log.isLoggable(Level.WARNING)) { log.log(Level.WARNING, ex.getMessage()); } - return null; + throw GcsHelperException.translate(ex); + } + } + + private static class DeleteBucketTask implements Callable { + + private Storage storage; + private String bucket; + + public DeleteBucketTask(Storage storage, String bucket) { + this.storage = storage; + this.bucket = bucket; + } + + @Override + public Boolean call() throws Exception { + while (true) { + for (BlobInfo info : storage.list(bucket)) { + storage.delete(bucket, info.name()); + } + try { + storage.delete(bucket); + return true; + } catch (StorageException e) { + if (e.code() == 409) { + Thread.sleep(500); + } else { + throw e; + } + } + } + } + } + + public static abstract class Option implements java.io.Serializable { + + private static final long serialVersionUID = 8849118657896662369L; + + public static final class KeyFromClasspath extends Option { + + private static final long serialVersionUID = -5506049413185246821L; + + private final boolean keyFromClasspath; + + public KeyFromClasspath(boolean keyFromClasspath) { + this.keyFromClasspath = keyFromClasspath; + } + + public boolean keyFromClasspath() { + return keyFromClasspath; + } + } + + Option() { + // package protected + } + + public static KeyFromClasspath keyFromClassPath() { + return new KeyFromClasspath(true); + } + + static Map, Option> asImmutableMap(Option... options) { + ImmutableMap.Builder, Option> builder = ImmutableMap.builder(); + for (Option option : options) { + builder.put(option.getClass(), option); + } + return builder.build(); + } + } + + public static class GcsHelperException extends RuntimeException { + + private static final long serialVersionUID = -7756074894502258736L; + + public GcsHelperException(String message) { + super(message); + } + + public GcsHelperException(String message, Throwable cause) { + super(message, cause); + } + + public static GcsHelperException translate(Exception ex) { + return new GcsHelperException(ex.getMessage(), ex); } } }