From 8dfbac0a3b3e0873f517d4eb096cb24efe3d52bd Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Mon, 26 Sep 2022 16:02:41 +0200 Subject: [PATCH 1/3] update the file copy strategy files for chmod -R --- .../nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy | 2 +- .../main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy | 2 +- .../cloud/google/batch/GoogleBatchScriptLauncher.groovy | 2 +- .../lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy index 36116e5174..a6be2aa2d1 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy @@ -71,7 +71,7 @@ class AwsBatchFileCopyStrategy extends SimpleFileCopyStrategy { // when a remote bin directory is provide managed it properly if( opts.remoteBinDir ) { result << "${opts.getAwsCli()} s3 cp --recursive --only-show-errors s3:/${opts.remoteBinDir} \$PWD/nextflow-bin\n" - result << "chmod +x \$PWD/nextflow-bin/*\n" + result << "chmod +x -R \$PWD/nextflow-bin/\n" result << "export PATH=\$PWD/nextflow-bin:\$PATH\n" } // finally render the environment diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy index 8ec6b35922..ab5e2f636b 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy @@ -87,7 +87,7 @@ class AzFileCopyStrategy extends SimpleFileCopyStrategy { String getStageInputFilesScript(Map inputFiles) { String result = ( remoteBinDir ? """\ nxf_az_download '${AzHelper.toHttpUrl(remoteBinDir)}' \$PWD/.nextflow-bin - chmod +x \$PWD/.nextflow-bin/* + chmod +x -R \$PWD/.nextflow-bin/ """.stripIndent() : '' ) result += 'downloads=(true)\n' diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy index bca544fa16..ab03f6e64e 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy @@ -89,7 +89,7 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder { def result = "NXF_CHDIR=${Escape.path(bean.workDir)}\n" if( remoteBinDir ) { result += "cp -r $remoteBinDir \$HOME/.nextflow-bin\n" - result += 'chmod +x $HOME/.nextflow-bin/*\n' + result += 'chmod +x -R $HOME/.nextflow-bin/\n' result += 'export PATH=$HOME/.nextflow-bin:$PATH\n' } return result diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy index 6981eabb09..0a923d700b 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy @@ -155,7 +155,7 @@ class GoogleLifeSciencesFileCopyStrategy extends SimpleFileCopyStrategy { copy.remove('PATH') // when a remote bin directory is provide managed it properly if( config.remoteBinDir ) { - result << "chmod +x $localTaskDir/nextflow-bin/*\n" + result << "chmod +x -R $localTaskDir/nextflow-bin/\n" result << "export PATH=$localTaskDir/nextflow-bin:\$PATH\n" } // finally render the environment From 38ae48fa57098c781183ace107fc68c03cbce2ce Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Mon, 26 Sep 2022 16:34:10 +0200 Subject: [PATCH 2/3] update relevant tests --- .../cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy | 6 +++--- .../cloud/aws/batch/AwsBatchScriptLauncherTest.groovy | 2 +- .../cloud/azure/batch/AzFileCopyStrategyTest.groovy | 2 +- .../GoogleLifeSciencesFileCopyStrategyTest.groovy | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy index 0c4fe1a1a1..57cb9c69a7 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy @@ -326,7 +326,7 @@ class AwsBatchFileCopyStrategyTest extends Specification { opts.getAwsCli() >> 'aws' script == ''' aws s3 cp --recursive --only-show-errors s3://foo/bar $PWD/nextflow-bin - chmod +x $PWD/nextflow-bin/* + chmod +x -R $PWD/nextflow-bin/ export PATH=$PWD/nextflow-bin:$PATH export BAR="world" export FOO="hola" @@ -339,7 +339,7 @@ class AwsBatchFileCopyStrategyTest extends Specification { opts.getRemoteBinDir() >> '/foo/bar' script == ''' /conda/bin/aws s3 cp --recursive --only-show-errors s3://foo/bar $PWD/nextflow-bin - chmod +x $PWD/nextflow-bin/* + chmod +x -R $PWD/nextflow-bin/ export PATH=$PWD/nextflow-bin:$PATH export BAR="world" export FOO="hola" @@ -353,7 +353,7 @@ class AwsBatchFileCopyStrategyTest extends Specification { opts.getRegion() >> 'eu-west-1' script == ''' /conda/bin/aws s3 cp --recursive --only-show-errors s3://foo/bar $PWD/nextflow-bin - chmod +x $PWD/nextflow-bin/* + chmod +x -R $PWD/nextflow-bin/ export PATH=$PWD/nextflow-bin:$PATH export BAR="world" export FOO="hola" diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy index b30a2a34d4..77d348bd12 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy @@ -159,7 +159,7 @@ class AwsBatchScriptLauncherTest extends Specification { then: binding.task_env == '''\ aws s3 cp --recursive --only-show-errors s3://bucket/bin $PWD/nextflow-bin - chmod +x $PWD/nextflow-bin/* + chmod +x -R $PWD/nextflow-bin/ export PATH=$PWD/nextflow-bin:$PATH export FOO="xxx" '''.stripIndent() diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy index 5b1c477a1a..39c18b0c7d 100644 --- a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy @@ -216,7 +216,7 @@ class AzFileCopyStrategyTest extends Specification { binding.stage_inputs == '''\ # stage input files nxf_az_download 'http://account.blob.core.windows.net/my-data/work/remote/bin' $PWD/.nextflow-bin - chmod +x $PWD/.nextflow-bin/* + chmod +x -R $PWD/.nextflow-bin/ downloads=(true) nxf_parallel "${downloads[@]}" diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategyTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategyTest.groovy index 1c1f562915..3b6633c336 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategyTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategyTest.groovy @@ -219,7 +219,7 @@ class GoogleLifeSciencesFileCopyStrategyTest extends GoogleSpecification { def envScript = strategy.getEnvScript([FOO:1, BAR: 2, PATH: 3], false) then: envScript == '''\ - chmod +x /work/xx/yy/nextflow-bin/* + chmod +x -R /work/xx/yy/nextflow-bin/ export PATH=/work/xx/yy/nextflow-bin:$PATH export BAR="2" export FOO="1" From 667289129048980c66d9d3a883c46998e87f7fa4 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Tue, 27 Sep 2022 09:42:35 +0200 Subject: [PATCH 3/3] accommodate code review --- .../cloud/aws/batch/AwsBatchFileCopyStrategy.groovy | 2 +- .../cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy | 6 +++--- .../cloud/aws/batch/AwsBatchScriptLauncherTest.groovy | 2 +- .../nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy | 2 +- .../cloud/azure/batch/AzFileCopyStrategyTest.groovy | 2 +- .../cloud/google/batch/GoogleBatchScriptLauncher.groovy | 2 +- .../lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy | 2 +- .../GoogleLifeSciencesFileCopyStrategyTest.groovy | 2 +- .../GoogleLifeSciencesScriptLauncherTest.groovy | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy index a6be2aa2d1..0ebe371c5d 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy @@ -71,7 +71,7 @@ class AwsBatchFileCopyStrategy extends SimpleFileCopyStrategy { // when a remote bin directory is provide managed it properly if( opts.remoteBinDir ) { result << "${opts.getAwsCli()} s3 cp --recursive --only-show-errors s3:/${opts.remoteBinDir} \$PWD/nextflow-bin\n" - result << "chmod +x -R \$PWD/nextflow-bin/\n" + result << "chmod +x \$PWD/nextflow-bin/* || true\n" result << "export PATH=\$PWD/nextflow-bin:\$PATH\n" } // finally render the environment diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy index 57cb9c69a7..a00331a018 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy @@ -326,7 +326,7 @@ class AwsBatchFileCopyStrategyTest extends Specification { opts.getAwsCli() >> 'aws' script == ''' aws s3 cp --recursive --only-show-errors s3://foo/bar $PWD/nextflow-bin - chmod +x -R $PWD/nextflow-bin/ + chmod +x $PWD/nextflow-bin/* || true export PATH=$PWD/nextflow-bin:$PATH export BAR="world" export FOO="hola" @@ -339,7 +339,7 @@ class AwsBatchFileCopyStrategyTest extends Specification { opts.getRemoteBinDir() >> '/foo/bar' script == ''' /conda/bin/aws s3 cp --recursive --only-show-errors s3://foo/bar $PWD/nextflow-bin - chmod +x -R $PWD/nextflow-bin/ + chmod +x $PWD/nextflow-bin/* || true export PATH=$PWD/nextflow-bin:$PATH export BAR="world" export FOO="hola" @@ -353,7 +353,7 @@ class AwsBatchFileCopyStrategyTest extends Specification { opts.getRegion() >> 'eu-west-1' script == ''' /conda/bin/aws s3 cp --recursive --only-show-errors s3://foo/bar $PWD/nextflow-bin - chmod +x -R $PWD/nextflow-bin/ + chmod +x $PWD/nextflow-bin/* || true export PATH=$PWD/nextflow-bin:$PATH export BAR="world" export FOO="hola" diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy index 77d348bd12..633e755cc0 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy @@ -159,7 +159,7 @@ class AwsBatchScriptLauncherTest extends Specification { then: binding.task_env == '''\ aws s3 cp --recursive --only-show-errors s3://bucket/bin $PWD/nextflow-bin - chmod +x -R $PWD/nextflow-bin/ + chmod +x $PWD/nextflow-bin/* || true export PATH=$PWD/nextflow-bin:$PATH export FOO="xxx" '''.stripIndent() diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy index ab5e2f636b..3476c184da 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy @@ -87,7 +87,7 @@ class AzFileCopyStrategy extends SimpleFileCopyStrategy { String getStageInputFilesScript(Map inputFiles) { String result = ( remoteBinDir ? """\ nxf_az_download '${AzHelper.toHttpUrl(remoteBinDir)}' \$PWD/.nextflow-bin - chmod +x -R \$PWD/.nextflow-bin/ + chmod +x \$PWD/.nextflow-bin/* || true """.stripIndent() : '' ) result += 'downloads=(true)\n' diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy index 39c18b0c7d..c50aac74c6 100644 --- a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy @@ -216,7 +216,7 @@ class AzFileCopyStrategyTest extends Specification { binding.stage_inputs == '''\ # stage input files nxf_az_download 'http://account.blob.core.windows.net/my-data/work/remote/bin' $PWD/.nextflow-bin - chmod +x -R $PWD/.nextflow-bin/ + chmod +x $PWD/.nextflow-bin/* || true downloads=(true) nxf_parallel "${downloads[@]}" diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy index ab03f6e64e..bca544fa16 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy @@ -89,7 +89,7 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder { def result = "NXF_CHDIR=${Escape.path(bean.workDir)}\n" if( remoteBinDir ) { result += "cp -r $remoteBinDir \$HOME/.nextflow-bin\n" - result += 'chmod +x -R $HOME/.nextflow-bin/\n' + result += 'chmod +x $HOME/.nextflow-bin/*\n' result += 'export PATH=$HOME/.nextflow-bin:$PATH\n' } return result diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy index 0a923d700b..6981eabb09 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy @@ -155,7 +155,7 @@ class GoogleLifeSciencesFileCopyStrategy extends SimpleFileCopyStrategy { copy.remove('PATH') // when a remote bin directory is provide managed it properly if( config.remoteBinDir ) { - result << "chmod +x -R $localTaskDir/nextflow-bin/\n" + result << "chmod +x $localTaskDir/nextflow-bin/*\n" result << "export PATH=$localTaskDir/nextflow-bin:\$PATH\n" } // finally render the environment diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategyTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategyTest.groovy index 3b6633c336..eef2d7d1dc 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategyTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategyTest.groovy @@ -219,7 +219,7 @@ class GoogleLifeSciencesFileCopyStrategyTest extends GoogleSpecification { def envScript = strategy.getEnvScript([FOO:1, BAR: 2, PATH: 3], false) then: envScript == '''\ - chmod +x -R /work/xx/yy/nextflow-bin/ + chmod +x /work/xx/yy/nextflow-bin/* || true export PATH=/work/xx/yy/nextflow-bin:$PATH export BAR="2" export FOO="1" diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy index 78d259587e..3f66ab4f3d 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy @@ -56,7 +56,7 @@ class GoogleLifeSciencesScriptLauncherTest extends GoogleSpecification { binding.stage_cmd == null binding.unstage_cmd == null binding.task_env == '''\ - chmod +x /work/xx/yy/nextflow-bin/* + chmod +x /work/xx/yy/nextflow-bin/* || true export PATH=/work/xx/yy/nextflow-bin:$PATH export FOO="xxx" '''.stripIndent()