Skip to content

Commit

Permalink
Integration test for ls (googleapis#931)
Browse files Browse the repository at this point in the history
  • Loading branch information
jean-philippe-martin authored and mziccard committed Jun 30, 2016
1 parent e874b76 commit 10180eb
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ private static class LazyPathIterator extends AbstractIterator<Path> {
private final Filter<? super Path> filter;
private final CloudStorageFileSystem fileSystem;

LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator, Filter<? super Path> filter) {
LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator,
Filter<? super Path> filter) {
this.blobIterator = blobIterator;
this.filter = filter;
this.fileSystem = fileSystem;
Expand Down Expand Up @@ -176,7 +177,8 @@ && isNullOrEmpty(uri.getUserInfo()),

@Override
public CloudStoragePath getPath(URI uri) {
return CloudStoragePath.getPath(getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
return CloudStoragePath.getPath(
getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
}

@Override
Expand Down Expand Up @@ -559,7 +561,9 @@ public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super P
final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir);
checkNotNull(filter);
String prefix = cloudPath.toString();
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll();
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(),
Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(),
Storage.BlobListOption.fields()).iterateAll();
return new DirectoryStream<Path>() {
@Override
public Iterator<Path> iterator() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,8 @@ public String toString() {
@Override
public URI toUri() {
try {
return new URI(CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null);
return new URI(
CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null);
} catch (URISyntaxException e) {
throw new AssertionError(e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,12 @@
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
Expand Down Expand Up @@ -316,6 +318,29 @@ public void testCopy() throws IOException {
}
}

@Test
public void testListFiles() throws IOException {
try (FileSystem fs = getTestBucket()) {
List<Path> goodPaths = new ArrayList<>();
List<Path> paths = new ArrayList<>();
goodPaths.add(fs.getPath("dir/angel"));
goodPaths.add(fs.getPath("dir/alone"));
paths.add(fs.getPath("dir/dir2/another_angel"));
paths.add(fs.getPath("atroot"));
paths.addAll(goodPaths);
goodPaths.add(fs.getPath("dir/dir2/"));
for (Path path : paths) {
fillFile(storage, path.toString(), SML_SIZE);
}

List<Path> got = new ArrayList<>();
for (Path path : Files.newDirectoryStream(fs.getPath("dir/"))) {
got.add(path);
}
assertThat(got).containsExactlyElementsIn(goodPaths);
}
}

private int readFully(ReadableByteChannel chan, byte[] outputBuf) throws IOException {
ByteBuffer buf = ByteBuffer.wrap(outputBuf);
int sofar = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) throws Stora
@Override
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
throws StorageException {
String delimiter = null;
String preprefix = "";
for (Map.Entry<Option, ?> e : options.entrySet()) {
switch (e.getKey()) {
Expand All @@ -103,6 +104,9 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
preprefix = preprefix.substring(1);
}
break;
case DELIMITER:
delimiter = (String) e.getValue();
break;
case FIELDS:
// ignore and return all the fields
break;
Expand All @@ -118,17 +122,7 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
if (!so.getName().startsWith(prefix)) {
continue;
}
int nextSlash = so.getName().indexOf("/", prefix.length());
if (nextSlash >= 0) {
String folderName = so.getName().substring(0, nextSlash + 1);
if (folders.containsKey(folderName)) {
continue;
}
StorageObject fakeFolder = new StorageObject();
fakeFolder.setName(folderName);
fakeFolder.setBucket(so.getBucket());
fakeFolder.setGeneration(so.getGeneration());
folders.put(folderName, fakeFolder);
if (processedAsFolder(so, delimiter, prefix, folders)) {
continue;
}
values.add(so);
Expand Down Expand Up @@ -334,4 +328,25 @@ private void potentiallyThrow(Map<Option, ?> options) throws UnsupportedOperatio
throw new UnsupportedOperationException();
}
}

// Returns true if this is a folder. Adds it to folders if it isn't already there.
private static boolean processedAsFolder(StorageObject so, String delimiter, String prefix, /* inout */ Map<String, StorageObject> folders) {
if (delimiter == null) {
return false;
}
int nextSlash = so.getName().indexOf(delimiter, prefix.length());
if (nextSlash < 0) {
return false;
}
String folderName = so.getName().substring(0, nextSlash + 1);
if (folders.containsKey(folderName)) {
return true;
}
StorageObject fakeFolder = new StorageObject();
fakeFolder.setName(folderName);
fakeFolder.setBucket(so.getBucket());
fakeFolder.setGeneration(so.getGeneration());
folders.put(folderName, fakeFolder);
return true;
}
}

0 comments on commit 10180eb

Please sign in to comment.