From 10180ebd9409ef37346325da86ee89892000162d Mon Sep 17 00:00:00 2001 From: JP Martin Date: Tue, 19 Apr 2016 13:56:54 -0700 Subject: [PATCH] Integration test for ls (#931) --- .../nio/CloudStorageFileSystemProvider.java | 10 +++-- .../storage/contrib/nio/CloudStoragePath.java | 3 +- .../storage/contrib/nio/it/ITGcsNio.java | 25 +++++++++++++ .../cloud/storage/testing/FakeStorageRpc.java | 37 +++++++++++++------ 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java index b9a226b7875a..f7ded91ecc61 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java @@ -88,7 +88,8 @@ private static class LazyPathIterator extends AbstractIterator { private final Filter filter; private final CloudStorageFileSystem fileSystem; - LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator blobIterator, Filter filter) { + LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator blobIterator, + Filter filter) { this.blobIterator = blobIterator; this.filter = filter; this.fileSystem = fileSystem; @@ -176,7 +177,8 @@ && isNullOrEmpty(uri.getUserInfo()), @Override public CloudStoragePath getPath(URI uri) { - return CloudStoragePath.getPath(getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath()); + return CloudStoragePath.getPath( + getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath()); } @Override @@ -559,7 +561,9 @@ public DirectoryStream newDirectoryStream(Path dir, final Filter blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll(); + final Iterator blobIterator = storage.list(cloudPath.bucket(), + Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.fields()).iterateAll(); return new DirectoryStream() { @Override public Iterator iterator() { diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java index 3f954efd3bee..fdb407ba1852 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java @@ -308,7 +308,8 @@ public String toString() { @Override public URI toUri() { try { - return new URI(CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null); + return new URI( + CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null); } catch (URISyntaxException e) { throw new AssertionError(e); } diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java index e5ead7d6ce37..0f2667ad2cad 100644 --- a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java @@ -27,10 +27,12 @@ import java.nio.ByteBuffer; import java.nio.channels.ReadableByteChannel; import java.nio.channels.SeekableByteChannel; +import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; @@ -316,6 +318,29 @@ public void testCopy() throws IOException { } } + @Test + public void testListFiles() throws IOException { + try (FileSystem fs = getTestBucket()) { + List goodPaths = new ArrayList<>(); + List paths = new ArrayList<>(); + goodPaths.add(fs.getPath("dir/angel")); + goodPaths.add(fs.getPath("dir/alone")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("atroot")); + paths.addAll(goodPaths); + goodPaths.add(fs.getPath("dir/dir2/")); + for (Path path : paths) { + fillFile(storage, path.toString(), SML_SIZE); + } + + List got = new ArrayList<>(); + for (Path path : Files.newDirectoryStream(fs.getPath("dir/"))) { + got.add(path); + } + assertThat(got).containsExactlyElementsIn(goodPaths); + } + } + private int readFully(ReadableByteChannel chan, byte[] outputBuf) throws IOException { ByteBuffer buf = ByteBuffer.wrap(outputBuf); int sofar = 0; diff --git a/gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java index 3b372d33a09a..24b38155368a 100644 --- a/gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java @@ -94,6 +94,7 @@ public Tuple> list(Map options) throws Stora @Override public Tuple> list(String bucket, Map options) throws StorageException { + String delimiter = null; String preprefix = ""; for (Map.Entry e : options.entrySet()) { switch (e.getKey()) { @@ -103,6 +104,9 @@ public Tuple> list(String bucket, Map preprefix = preprefix.substring(1); } break; + case DELIMITER: + delimiter = (String) e.getValue(); + break; case FIELDS: // ignore and return all the fields break; @@ -118,17 +122,7 @@ public Tuple> list(String bucket, Map if (!so.getName().startsWith(prefix)) { continue; } - int nextSlash = so.getName().indexOf("/", prefix.length()); - if (nextSlash >= 0) { - String folderName = so.getName().substring(0, nextSlash + 1); - if (folders.containsKey(folderName)) { - continue; - } - StorageObject fakeFolder = new StorageObject(); - fakeFolder.setName(folderName); - fakeFolder.setBucket(so.getBucket()); - fakeFolder.setGeneration(so.getGeneration()); - folders.put(folderName, fakeFolder); + if (processedAsFolder(so, delimiter, prefix, folders)) { continue; } values.add(so); @@ -334,4 +328,25 @@ private void potentiallyThrow(Map options) throws UnsupportedOperatio throw new UnsupportedOperationException(); } } + + // Returns true if this is a folder. Adds it to folders if it isn't already there. + private static boolean processedAsFolder(StorageObject so, String delimiter, String prefix, /* inout */ Map folders) { + if (delimiter == null) { + return false; + } + int nextSlash = so.getName().indexOf(delimiter, prefix.length()); + if (nextSlash < 0) { + return false; + } + String folderName = so.getName().substring(0, nextSlash + 1); + if (folders.containsKey(folderName)) { + return true; + } + StorageObject fakeFolder = new StorageObject(); + fakeFolder.setName(folderName); + fakeFolder.setBucket(so.getBucket()); + fakeFolder.setGeneration(so.getGeneration()); + folders.put(folderName, fakeFolder); + return true; + } }