diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index a0f1448d72..ecc67d9ae3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -19,11 +19,14 @@ package nextflow.processor import java.nio.file.FileAlreadyExistsException import java.nio.file.FileSystem import java.nio.file.FileSystems +import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.LinkOption import java.nio.file.NoSuchFileException import java.nio.file.Path import java.nio.file.PathMatcher +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes import java.time.temporal.ChronoUnit import java.util.concurrent.ExecutorService @@ -337,10 +340,10 @@ class PublishDir { } if( inProcess ) { - safeProcessFile(source, destination) + safeProcessPath(source, destination) } else { - threadPool.submit({ safeProcessFile(source, destination) } as Runnable) + threadPool.submit({ safeProcessPath(source, destination) } as Runnable) } } @@ -363,9 +366,23 @@ class PublishDir { throw new IllegalArgumentException("Not a valid publish target path: `$target` [${target?.class?.name}]") } - protected void safeProcessFile(Path source, Path target) { + protected void safeProcessPath(Path source, Path target) { try { - retryableProcessFile(source, target) + // publish each file in the directory tree + if( Files.isDirectory(source) ) { + Files.walkFileTree(source, new SimpleFileVisitor() { + FileVisitResult visitFile(Path sourceFile, BasicFileAttributes attrs) { + final targetFile = target.resolve(source.relativize(sourceFile).toString()) + retryableProcessFile(sourceFile, targetFile) + FileVisitResult.CONTINUE + } + }) + } + + // otherwise publish file directly + else { + retryableProcessFile(source, target) + } } catch( Throwable e ) { final msg = "Failed to publish file: ${source.toUriString()}; to: ${target.toUriString()} [${mode.toString().toLowerCase()}] -- See log file for details" @@ -395,7 +412,7 @@ class PublishDir { .build() Failsafe .with( retryPolicy ) - .get({it-> processFile(source, target)}) + .get { it-> processFile(source, target) } } protected void processFile( Path source, Path destination ) { diff --git a/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy b/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy index adeb76cd43..fe9083d314 100644 --- a/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy +++ b/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy @@ -63,7 +63,7 @@ class PublishDirS3Test extends Specification { when: spy.apply1(source, true) then: - 1 * spy.safeProcessFile(source, _) >> { sourceFile, s3File -> + 1 * spy.safeProcessPath(source, _) >> { sourceFile, s3File -> assert s3File instanceof S3Path assert (s3File as S3Path).getTagsList().find{ it.getKey()=='FOO'}.value == 'this' assert (s3File as S3Path).getTagsList().find{ it.getKey()=='BAR'}.value == 'that'