Add support for custom custom root directory to resolve relative paths

docs/config.md

@@ -1601,6 +1601,11 @@ The following environment variables control the configuration of the Nextflow ru
 `NXF_WORK`
 : Directory where working files are stored (usually your *scratch* directory)
 
+`NXF_FILE_ROOT`
+: The file storage path against which relative file paths are resolved. For example, having define the variable `NXF_FILE_ROOT=/some/root/path`
+the use of `file('foo')` will be resolved to the absolute path `/some/root/path/foo`. A remote root path can be specified using the 
+usual protocol prefix e.g. `NXF_FILE_ROOT=s3://my-bucket/data`. Files defined using an absolute path are not affected by this setting.
+
 `JAVA_HOME`
 : Defines the path location of the Java VM installation used to run Nextflow.
 

modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy

@@ -16,7 +16,8 @@
 
 package nextflow
 
-import java.nio.file.FileSystem
+import static nextflow.file.FileHelper.*
+
 import java.nio.file.Files
 import java.nio.file.NoSuchFileException
 import java.nio.file.Path
@@ -39,7 +40,6 @@ import nextflow.util.ArrayTuple
 import nextflow.util.CacheHelper
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
-import static nextflow.file.FileHelper.isGlobAllowed
 /**
  * Defines the main methods imported by default in the script scope
  *
@@ -57,21 +57,18 @@ class Nextflow {
     private static final Random random = new Random()
 
 
-    static private fileNamePattern( FilePatternSplitter splitter, Map opts, FileSystem fs ) {
+    static private fileNamePattern( FilePatternSplitter splitter, Map opts ) {
 
         final scheme = splitter.scheme
-        final folder = splitter.parent
+        final folder = toCanonicalPath(splitter.parent)
         final pattern = splitter.fileName
 
-        if( !fs )
-            fs = FileHelper.fileSystemForScheme(scheme)
-
         if( opts == null ) opts = [:]
         if( !opts.type ) opts.type = 'file'
 
         def result = new LinkedList()
         try {
-            FileHelper.visitFiles(opts, fs.getPath(folder), pattern) { Path it -> result.add(it) }
+            FileHelper.visitFiles(opts, folder, pattern) { Path it -> result.add(it) }
         }
         catch (NoSuchFileException e) {
             log.debug "No such file or directory: $folder -- Skipping visit"
@@ -80,6 +77,18 @@ class Nextflow {
 
     }
 
+    static private String str0(value) {
+        if( value==null )
+            return null
+        if( value instanceof CharSequence )
+            return value.toString()
+        if( value instanceof File )
+            return value.toString()
+        if( value instanceof Path )
+            return value.toUriString()
+        throw new IllegalArgumentException("Invalid file path type - offending value: $value [${value.getClass().getName()}]")
+    }
+
     /**
      * Get one or more file object given the specified path or glob pattern.
      *
@@ -101,23 +110,19 @@ class Nextflow {
         final path = filePattern as Path
         final glob = options?.containsKey('glob') ? options.glob as boolean : isGlobAllowed(path)
         if( !glob ) {
-            return FileHelper.checkIfExists(path, options)
+            return checkIfExists(toCanonicalPath(path), options)
         }
 
         // if it isn't a glob pattern simply return it a normalized absolute Path object
-        def splitter = FilePatternSplitter.glob().parse(path.toString())
+        final strPattern = str0(filePattern)
+        final splitter = FilePatternSplitter.glob().parse(strPattern)
         if( !splitter.isPattern() ) {
-            def normalised = splitter.strip(path.toString())
-            if( path instanceof Path )  {
-                return FileHelper.checkIfExists(path.fileSystem.getPath(normalised), options)
-            }
-            else {
-                return FileHelper.checkIfExists(FileHelper.asPath(normalised), options)
-            }
+            final normalised = splitter.strip(strPattern)
+            return checkIfExists(toCanonicalPath(normalised), options)
         }
 
         // revolve the glob pattern returning all matches
-        return fileNamePattern(splitter, options, path.getFileSystem())
+        return fileNamePattern(splitter, options)
     }
 
     static files( Map options=null, def path ) {

modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy

@@ -122,17 +122,11 @@ class PublishDir {
     @Lazy
     private ExecutorService threadPool = { def sess = Global.session as Session; sess.publishDirExecutorService() }()
 
-    void setPath( Closure obj ) {
-        setPath( obj.call() as Path )
-    }
-
-    void setPath( String str ) {
-        nullPathWarn = checkNull(str)
-        setPath(str as Path)
-    }
-
-    void setPath( Path obj ) {
-        this.path = obj.complete()
+    void setPath( def value ) {
+        final resolved = value instanceof Closure ? value.call() : value
+        if( resolved instanceof String || resolved instanceof GString )
+            nullPathWarn = checkNull(resolved.toString())
+        this.path = FileHelper.toCanonicalPath(resolved)
     }
 
     void setMode( String str ) {

modules/nextflow/src/main/groovy/nextflow/util/PathSplitter.groovy

@@ -20,6 +20,7 @@ package nextflow.util
 import groovy.transform.Canonical
 import groovy.transform.CompileStatic
 import groovy.transform.EqualsAndHashCode
+import nextflow.file.FileHelper
 
 /**
  * Split a path into two paths, the first component which may include the host name if it's a remote
@@ -35,7 +36,7 @@ class PathSplitter {
     List<String> tail
 
     static PathSplitter parse(String path) {
-        final baseUrl = StringUtils.baseUrl(path)
+        final baseUrl = FileHelper.baseUrl(path)
         if( !baseUrl )
             return split0(path, 0)
 

modules/nextflow/src/test/groovy/nextflow/NextflowTest.groovy

@@ -15,6 +15,7 @@
  */
 
 package nextflow
+
 import java.nio.file.Files
 import java.nio.file.NoSuchFileException
 import java.nio.file.Paths
@@ -35,12 +36,10 @@ class NextflowTest extends Specification {
     }
 
     def testFile() {
-
         expect:
         Nextflow.file('file.log').toFile() == new File('file.log').canonicalFile
         Nextflow.file('relative/file.test').toFile() == new File( new File('.').canonicalFile, 'relative/file.test')
         Nextflow.file('/user/home/file.log').toFile() == new File('/user/home/file.log')
-
     }
 
     def testFile2() {
@@ -56,6 +55,19 @@ class NextflowTest extends Specification {
 
     }
 
+    def 'should resolve rel paths against env base' () {
+        given:
+        SysEnv.push(NXF_FILE_ROOT: '/some/base/dir')
+
+        expect:
+        Nextflow.file( '/abs/path/file.txt' ) == Paths.get('/abs/path/file.txt')
+        and:
+        Nextflow.file( 'file.txt' ) == Paths.get('/some/base/dir/file.txt')
+
+        cleanup:
+        SysEnv.pop()
+    }
+
     def testFile3() {
         Exception e
         when:

modules/nextflow/src/test/groovy/nextflow/util/PathSplitterTest.groovy

@@ -43,6 +43,12 @@ class PathSplitterTest extends Specification {
         '/foo/bar/baz'  | new PathSplitter('/foo', ['bar','baz'])
         'foo/bar/baz/'  | new PathSplitter('foo', ['bar','baz'])
         '/foo/bar/baz/' | new PathSplitter('/foo', ['bar','baz'])
+        '/foo/x/y/z'    | new PathSplitter('/foo', ['x','y','z'])
+        and:
+        'file:/foo'                     | new PathSplitter('file:/foo', null)
+        'file:/foo/x/y/z'               | new PathSplitter('file:/foo', ['x','y','z'])
+        'file:///foo'                   | new PathSplitter('file:///foo', null)
+        'file:///foo/x/y/z'             | new PathSplitter('file:///foo', ['x','y','z'])
         and:
         's3://my-bucket'                | new PathSplitter('s3://my-bucket')
         's3://my-bucket/'               | new PathSplitter('s3://my-bucket/')

modules/nf-commons/src/main/nextflow/file/FileHelper.groovy

@@ -43,6 +43,7 @@ import groovy.transform.Memoized
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import nextflow.Global
+import nextflow.SysEnv
 import nextflow.extension.Bolts
 import nextflow.extension.FilesEx
 import nextflow.plugin.Plugins
@@ -57,7 +58,9 @@ import nextflow.util.Escape
 @CompileStatic
 class FileHelper {
 
-    static final public Pattern URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*)\:\\/\\/.+/
+    static final public Pattern URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*):\\/\\/.+/
+
+    static final private Pattern BASE_URL = ~/(?i)((?:[a-z][a-zA-Z0-9]*)?:\/\/[^:|\/]+(?::\d*)?)(?:$|\/.*)/
 
     static final private Path localTempBasePath
 
@@ -233,6 +236,34 @@ class FileHelper {
         return !(path.getFileSystem().provider().scheme in UNSUPPORTED_GLOB_WILDCARDS)
     }
 
+    static Path toCanonicalPath(value) {
+        if( value==null )
+            return null
+
+        Path result = null
+        if( value instanceof String || value instanceof GString ) {
+            result = asPath(value.toString())
+        }
+        else if( value instanceof Path ) {
+            result = (Path)value
+        }
+        else {
+            throw new IllegalArgumentException("Unexpected path value: '$value' [${value.getClass().getName()}]")
+        }
+
+        if( result.fileSystem != FileSystems.default ) {
+            // remote file paths are expected to be absolute by definition
+            return result
+        }
+
+        Path base
+        if( !result.isAbsolute() && (base=fileRootDir()) ) {
+            result = base.resolve(result.toString())
+        }
+
+        return result.toAbsolutePath().normalize()
+    }
+
     /**
      * Given an hierarchical file URI path returns a {@link Path} object
      * eventually creating the associated file system if required.
@@ -257,6 +288,10 @@ class FileHelper {
             return Paths.get(str)
         }
 
+        return asPath0(str)
+    }
+
+    static private Path asPath0(String str) {
         def result = FileSystemPathFactory.parse(str)
         if( result )
             return result
@@ -273,7 +308,7 @@ class FileHelper {
         return asPath(toPathURI(str))
     }
 
-    static private Map<String,String> PLUGINS_MAP = [s3:'nf-amazon', gs:'nf-google', az:'nf-azure']
+    static final private Map<String,String> PLUGINS_MAP = [s3:'nf-amazon', gs:'nf-google', az:'nf-azure']
 
     static final private Map<String,Boolean> SCHEME_CHECKED = new HashMap<>()
 
@@ -1042,8 +1077,44 @@ class FileHelper {
     }
 
     static String getUrlProtocol(String str) {
+        if( !str )
+            return null
+        // note: `file:/foo` is a valid file pseudo-protocol, and represents absolute
+        // `/foo` path with no remote hostname specified
+        if( str.startsWith('file:/'))
+            return 'file'
         final m = URL_PROTOCOL.matcher(str)
         return m.matches() ? m.group(1) : null
     }
 
+    static Path fileRootDir() {
+        if( !SysEnv.get('NXF_FILE_ROOT') )
+            return null
+        final base = SysEnv.get('NXF_FILE_ROOT')
+        if( base.startsWith('/') )
+            return Paths.get(base)
+        final scheme = getUrlProtocol(base)
+        if( !scheme )
+            throw new IllegalArgumentException("Invalid NXF_FILE_ROOT environment value - It must be an absolute path or a valid path URI - Offending value: '$base'")
+        return asPath0(base)
+    }
+
+    static String baseUrl(String url) {
+        if( !url )
+            return null
+        final m = BASE_URL.matcher(url)
+        if( m.matches() )
+            return m.group(1).toLowerCase()
+        if( url.startsWith('file:///')) {
+            return 'file:///'
+        }
+        if( url.startsWith('file://')) {
+            return url.length()>7 ? url.substring(7).tokenize('/')[0] : null
+        }
+        if( url.startsWith('file:/')) {
+            return 'file:/'
+        }
+        return null
+    }
+
 }

modules/nf-commons/src/main/nextflow/util/StringUtils.groovy

@@ -22,6 +22,7 @@ import java.util.regex.Pattern
 
 import com.google.common.net.InetAddresses
 import groovy.transform.CompileStatic
+
 /**
  * String helper routines
  *
@@ -33,13 +34,21 @@ class StringUtils {
     static final public Pattern URL_PROTOCOL = ~/^([a-zA-Z0-9]*):\\/\\/(.+)/
     static final private Pattern URL_PASSWORD = ~/^[a-zA-Z][a-zA-Z0-9]*:\\/\\/(.+)@.+/
 
+    /**
+     * Deprecated. Use {@link nextflow.file.FileHelper#getUrlProtocol(java.lang.String)} instead
+     */
+    @Deprecated
     static String getUrlProtocol(String str) {
         final m = URL_PROTOCOL.matcher(str)
         return m.matches() ? m.group(1) : null
     }
 
     static final private Pattern BASE_URL = ~/(?i)((?:[a-z][a-zA-Z0-9]*)?:\/\/[^:|\/]+(?::\d*)?)(?:$|\/.*)/
 
+    /**
+     * Deprecated. Use {@link nextflow.file.FileHelper#baseUrl(java.lang.String)} instead
+     */
+    @Deprecated
     static String baseUrl(String url) {
         if( !url )
             return null