From 9c9dc0d07e4183ed824e642b28eafc70313dfbc8 Mon Sep 17 00:00:00 2001 From: JP Martin Date: Thu, 14 Apr 2016 13:24:53 -0700 Subject: [PATCH] Implement bare-bones blob listing --- .../nio/CloudStorageFileSystemProvider.java | 54 ++++++++++++++++--- .../nio/CloudStorageFileSystemTest.java | 26 +++++++++ .../nio/CloudStorageReadChannelTest.java | 2 - .../storage/testing/FakeStorageRpc.java | 49 +++++++++++++++-- 4 files changed, 119 insertions(+), 12 deletions(-) diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java index 4c07a5eb22fe..5fbcf754cc76 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java @@ -29,8 +29,10 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.base.Throwables; +import com.google.common.collect.AbstractIterator; import com.google.common.primitives.Ints; import com.google.gcloud.storage.Acl; +import com.google.gcloud.storage.Blob; import com.google.gcloud.storage.BlobId; import com.google.gcloud.storage.BlobInfo; import com.google.gcloud.storage.CopyWriter; @@ -46,6 +48,7 @@ import java.nio.file.AccessMode; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.CopyOption; +import java.nio.file.DirectoryIteratorException; import java.nio.file.DirectoryStream; import java.nio.file.DirectoryStream.Filter; import java.nio.file.FileAlreadyExistsException; @@ -64,11 +67,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; - import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; @@ -84,6 +87,33 @@ public final class CloudStorageFileSystemProvider extends FileSystemProvider { // used only when we create a new instance of CloudStorageFileSystemProvider. private static StorageOptions defaultStorageOptions; + private static class LazyPathIterator extends AbstractIterator { + private final Iterator blobIterator; + private final Filter filter; + private final CloudStorageFileSystem fileSystem; + + LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator blobIterator, Filter filter) { + this.blobIterator = blobIterator; + this.filter = filter; + this.fileSystem = fileSystem; + } + + @Override + protected Path computeNext() { + while (blobIterator.hasNext()) { + Path path = fileSystem.getPath(blobIterator.next().name()); + try { + if (filter.accept(path)) { + return path; + } + } catch (IOException ex) { + throw new DirectoryIteratorException(ex); + } + } + return endOfData(); + } + } + /** * Sets default options that are only used by the constructor. */ @@ -532,13 +562,23 @@ public void createDirectory(Path dir, FileAttribute... attrs) { checkNotNullArray(attrs); } - /** - * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. - */ @Override - public DirectoryStream newDirectoryStream(Path dir, Filter filter) { - // TODO: Implement me. - throw new UnsupportedOperationException(); + public DirectoryStream newDirectoryStream(Path dir, final Filter filter) { + final CloudStoragePath cloudPath = checkPath(dir); + checkNotNull(filter); + String prefix = cloudPath.toString(); + final Iterator blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll(); + return new DirectoryStream() { + @Override + public Iterator iterator() { + return new LazyPathIterator(cloudPath.getFileSystem(), blobIterator, filter); + } + + @Override + public void close() throws IOException { + // Does nothing since there's nothing to close. Commenting this method to quiet codacy. + } + }; } /** diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java index aacdb24268c6..f20ad6369ea7 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java @@ -34,7 +34,10 @@ import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; /** * Unit tests for {@link CloudStorageFileSystem}. @@ -134,4 +137,27 @@ public void testNullness() throws IOException, NoSuchMethodException, SecurityEx tester.testAllPublicInstanceMethods(fs); } } + + @Test + public void testListFiles() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + List goodPaths = new ArrayList<>(); + List paths = new ArrayList<>(); + goodPaths.add(fs.getPath("dir/angel")); + goodPaths.add(fs.getPath("dir/alone")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("atroot")); + paths.addAll(goodPaths); + goodPaths.add(fs.getPath("dir/dir2/")); + for (Path path : paths) { + Files.write(path, ALONE.getBytes(UTF_8)); + } + + List got = new ArrayList<>(); + for (Path path : Files.newDirectoryStream(fs.getPath("/dir/"))) { + got.add(path); + } + assertThat(got).containsExactlyElementsIn(goodPaths); + } + } } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java index be6cf58e24a8..82940095d972 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java @@ -22,14 +22,12 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import com.google.gcloud.ReadChannel; import com.google.gcloud.storage.Blob; import com.google.gcloud.storage.BlobId; -import com.google.gcloud.storage.BlobInfo; import com.google.gcloud.storage.Storage; import org.junit.Before; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/FakeStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/FakeStorageRpc.java index b874a467f962..01e07a46d4f4 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/FakeStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/testing/FakeStorageRpc.java @@ -26,8 +26,10 @@ import java.io.InputStream; import java.math.BigInteger; import java.nio.file.FileAlreadyExistsException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.annotation.concurrent.NotThreadSafe; @@ -91,8 +93,49 @@ public Tuple> list(Map options) throws Stora @Override public Tuple> list(String bucket, Map options) throws StorageException { - potentiallyThrow(options); - return null; + String preprefix = ""; + for (Map.Entry e : options.entrySet()) { + switch (e.getKey()) { + case PREFIX: + preprefix = (String) e.getValue(); + if (preprefix.startsWith("/")) { + preprefix = preprefix.substring(1); + } + break; + case FIELDS: + // ignore and return all the fields + break; + default: + throw new UnsupportedOperationException("Unknown option: " + e.getKey()); + } + } + final String prefix = preprefix; + + List values = new ArrayList<>(); + Map folders = new HashMap<>(); + for (StorageObject so : stuff.values()) { + if (!so.getName().startsWith(prefix)) { + continue; + } + int nextSlash = so.getName().indexOf("/", prefix.length()); + if (nextSlash >= 0) { + String folderName = so.getName().substring(0, nextSlash + 1); + if (folders.containsKey(folderName)) { + continue; + } + StorageObject fakeFolder = new StorageObject(); + fakeFolder.setName(folderName); + fakeFolder.setBucket(so.getBucket()); + fakeFolder.setGeneration(so.getGeneration()); + folders.put(folderName, fakeFolder); + continue; + } + values.add(so); + } + values.addAll(folders.values()); + // null cursor to indicate there is no more data (empty string would cause us to be called again). + // The type cast seems to be necessary to help Java's typesystem remember that collections are iterable. + return Tuple.of(null, (Iterable) values); } /** @@ -111,7 +154,7 @@ public Bucket get(Bucket bucket, Map options) throws StorageException public StorageObject get(StorageObject object, Map options) throws StorageException { // we allow the "ID" option because we need to, but then we give a whole answer anyways // because the caller won't mind the extra fields. - if (throwIfOption && !options.isEmpty() && options.size()>1 + if (throwIfOption && !options.isEmpty() && options.size() > 1 && options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) { throw new UnsupportedOperationException(); }