Skip to content

Commit

Permalink
Merge pull request #704 from mziccard/fix-storage-its
Browse files Browse the repository at this point in the history
Handle eventally consistent blob lists in storage ITs
  • Loading branch information
mziccard committed Mar 3, 2016
2 parents be913a6 + 622dcc9 commit 50750d4
Showing 1 changed file with 46 additions and 16 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.gcloud.Page;
import com.google.gcloud.ReadChannel;
Expand Down Expand Up @@ -100,13 +101,12 @@ public static void afterClass() throws ExecutionException, InterruptedException

@Test(timeout = 5000)
public void testListBuckets() throws InterruptedException {
Iterator<Bucket> bucketIterator =
storage.list(Storage.BucketListOption.prefix(BUCKET),
Storage.BucketListOption.fields()).values().iterator();
Iterator<Bucket> bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET),
Storage.BucketListOption.fields()).iterateAll();
while (!bucketIterator.hasNext()) {
Thread.sleep(500);
bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET),
Storage.BucketListOption.fields()).values().iterator();
Storage.BucketListOption.fields()).iterateAll();
}
while (bucketIterator.hasNext()) {
Bucket remoteBucket = bucketIterator.next();
Expand Down Expand Up @@ -287,8 +287,8 @@ public void testGetBlobFailNonExistingGeneration() {
assertTrue(remoteBlob.delete());
}

@Test
public void testListBlobsSelectedFields() {
@Test(timeout = 5000)
public void testListBlobsSelectedFields() throws InterruptedException {
String[] blobNames = {"test-list-blobs-selected-fields-blob1",
"test-list-blobs-selected-fields-blob2"};
ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v");
Expand All @@ -307,19 +307,29 @@ public void testListBlobsSelectedFields() {
Page<Blob> page = storage.list(BUCKET,
Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"),
Storage.BlobListOption.fields(BlobField.METADATA));
int index = 0;
for (Blob remoteBlob : page.values()) {
// Listing blobs is eventually consistent, we loop until the list is of the expected size. The
// test fails if timeout is reached.
while (Iterators.size(page.iterateAll()) != 2) {
Thread.sleep(500);
page = storage.list(BUCKET,
Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"),
Storage.BlobListOption.fields(BlobField.METADATA));
}
Set<String> blobSet = ImmutableSet.of(blobNames[0], blobNames[1]);
Iterator<Blob> iterator = page.iterateAll();
while (iterator.hasNext()) {
Blob remoteBlob = iterator.next();
assertEquals(BUCKET, remoteBlob.bucket());
assertEquals(blobNames[index++], remoteBlob.name());
assertTrue(blobSet.contains(remoteBlob.name()));
assertEquals(metadata, remoteBlob.metadata());
assertNull(remoteBlob.contentType());
}
assertTrue(remoteBlob1.delete());
assertTrue(remoteBlob2.delete());
}

@Test
public void testListBlobsEmptySelectedFields() {
@Test(timeout = 5000)
public void testListBlobsEmptySelectedFields() throws InterruptedException {
String[] blobNames = {"test-list-blobs-empty-selected-fields-blob1",
"test-list-blobs-empty-selected-fields-blob2"};
BlobInfo blob1 = BlobInfo.builder(BUCKET, blobNames[0])
Expand All @@ -335,17 +345,27 @@ public void testListBlobsEmptySelectedFields() {
Page<Blob> page = storage.list(BUCKET,
Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"),
Storage.BlobListOption.fields());
int index = 0;
for (Blob remoteBlob : page.values()) {
// Listing blobs is eventually consistent, we loop until the list is of the expected size. The
// test fails if timeout is reached.
while (Iterators.size(page.iterateAll()) != 2) {
Thread.sleep(500);
page = storage.list(BUCKET,
Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"),
Storage.BlobListOption.fields());
}
Set<String> blobSet = ImmutableSet.of(blobNames[0], blobNames[1]);
Iterator<Blob> iterator = page.iterateAll();
while (iterator.hasNext()) {
Blob remoteBlob = iterator.next();
assertEquals(BUCKET, remoteBlob.bucket());
assertEquals(blobNames[index++], remoteBlob.name());
assertTrue(blobSet.contains(remoteBlob.name()));
assertNull(remoteBlob.contentType());
}
assertTrue(remoteBlob1.delete());
assertTrue(remoteBlob2.delete());
}

@Test
@Test(timeout = 15000)
public void testListBlobsVersioned() throws ExecutionException, InterruptedException {
String bucketName = RemoteGcsHelper.generateBucketName();
Bucket bucket = storage.create(BucketInfo.builder(bucketName).versioningEnabled(true).build());
Expand All @@ -366,8 +386,18 @@ public void testListBlobsVersioned() throws ExecutionException, InterruptedExcep
Page<Blob> page = storage.list(bucketName,
Storage.BlobListOption.prefix("test-list-blobs-versioned-blob"),
Storage.BlobListOption.versions(true));
// Listing blobs is eventually consistent, we loop until the list is of the expected size. The
// test fails if timeout is reached.
while (Iterators.size(page.iterateAll()) != 3) {
Thread.sleep(500);
page = storage.list(bucketName,
Storage.BlobListOption.prefix("test-list-blobs-versioned-blob"),
Storage.BlobListOption.versions(true));
}
Set<String> blobSet = ImmutableSet.of(blobNames[0], blobNames[1]);
for (Blob remoteBlob : page.values()) {
Iterator<Blob> iterator = page.iterateAll();
while (iterator.hasNext()) {
Blob remoteBlob = iterator.next();
assertEquals(bucketName, remoteBlob.bucket());
assertTrue(blobSet.contains(remoteBlob.name()));
assertNotNull(remoteBlob.generation());
Expand Down

0 comments on commit 50750d4

Please sign in to comment.