diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy index 25ab24a205..d79ec957e3 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy @@ -134,8 +134,7 @@ class AwsBatchFileCopyStrategy extends SimpleFileCopyStrategy { */ @Override String touchFile( Path file ) { - final aws = opts.getAwsCli() - "echo start | $aws s3 cp --only-show-errors - s3:/${Escape.path(file)}" + "echo start | nxf_s3_upload - s3:/${Escape.path(file)}" } /** @@ -162,8 +161,7 @@ class AwsBatchFileCopyStrategy extends SimpleFileCopyStrategy { * {@inheritDoc} */ String exitFile( Path path ) { - final aws = opts.getAwsCli() - "| $aws s3 cp --only-show-errors - s3:/${Escape.path(path)} || true" + "| nxf_s3_upload - s3:/${Escape.path(path)} || true" } /** diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy index ad549ce2da..8f075178de 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy @@ -28,7 +28,7 @@ import nextflow.executor.BashFunLib class S3BashLib extends BashFunLib { private String storageClass = 'STANDARD' - private String encryptionEncryption = '' + private String storageEncryption = '' private String debug = '' private String cli = 'aws' private String retryMode @@ -58,7 +58,7 @@ class S3BashLib extends BashFunLib { S3BashLib withStorageEncryption(String value) { if( value ) - this.encryptionEncryption = value ? "--sse $value " : '' + this.storageEncryption = value ? "--sse $value " : '' return this } @@ -78,10 +78,12 @@ class S3BashLib extends BashFunLib { nxf_s3_upload() { local name=\$1 local s3path=\$2 - if [[ -d "\$name" ]]; then - $cli s3 cp --only-show-errors --recursive $debug$encryptionEncryption--storage-class $storageClass "\$name" "\$s3path/\$name" + if [[ "\$name" == - ]]; then + $cli s3 cp --only-show-errors $debug$storageEncryption--storage-class $storageClass - "\$s3path" + elif [[ -d "\$name" ]]; then + $cli s3 cp --only-show-errors --recursive $debug$storageEncryption--storage-class $storageClass "\$name" "\$s3path/\$name" else - $cli s3 cp --only-show-errors $debug$encryptionEncryption--storage-class $storageClass "\$name" "\$s3path/\$name" + $cli s3 cp --only-show-errors $debug$storageEncryption--storage-class $storageClass "\$name" "\$s3path/\$name" fi } diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy index 047baefeeb..545bf64a2d 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy @@ -33,9 +33,9 @@ class AwsBatchFileCopyStrategyTest extends Specification { def RUN = Paths.get('/some/data/.command.run') def copy = new AwsBatchFileCopyStrategy(Mock(TaskBean), new AwsOptions()) expect: - copy.touchFile(RUN) == "echo start | aws s3 cp --only-show-errors - s3://some/data/.command.run" + copy.touchFile(RUN) == "echo start | nxf_s3_upload - s3://some/data/.command.run" copy.copyFile("nobel_prize_results.gz",Paths.get("/some/data/nobel_prize_results.gz")) == "nxf_s3_upload nobel_prize_results.gz s3://some/data" - copy.exitFile(EXIT) == "| aws s3 cp --only-show-errors - s3://some/path/.exitcode || true" + copy.exitFile(EXIT) == "| nxf_s3_upload - s3://some/path/.exitcode || true" copy.stageInputFile(FILE, 'foo.txt') == """ downloads+=("nxf_s3_download s3://some/data/nobel_prize_results.gz foo.txt") """ @@ -188,7 +188,9 @@ class AwsBatchFileCopyStrategyTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -275,7 +277,9 @@ class AwsBatchFileCopyStrategyTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + /foo/aws s3 cp --only-show-errors --sse AES256 --storage-class STANDARD_IA - "$s3path" + elif [[ -d "$name" ]]; then /foo/aws s3 cp --only-show-errors --recursive --sse AES256 --storage-class STANDARD_IA "$name" "$s3path/$name" else /foo/aws s3 cp --only-show-errors --sse AES256 --storage-class STANDARD_IA "$name" "$s3path/$name" diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy index 00658ac8b7..7b06de8d2a 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy @@ -117,7 +117,9 @@ class AwsBatchScriptLauncherTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + /conda/bin/aws --region eu-west-1 s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then /conda/bin/aws --region eu-west-1 s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else /conda/bin/aws --region eu-west-1 s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -292,7 +294,9 @@ class AwsBatchScriptLauncherTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -433,7 +437,9 @@ class AwsBatchScriptLauncherTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -548,7 +554,9 @@ class AwsBatchScriptLauncherTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy index 25cec34f8c..20058a7f7b 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy @@ -86,7 +86,9 @@ class S3BashLibTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -181,7 +183,9 @@ class S3BashLibTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + /foo/bin/aws s3 cp --only-show-errors --sse S-ENCRYPT --storage-class S-CLAZZ - "$s3path" + elif [[ -d "$name" ]]; then /foo/bin/aws s3 cp --only-show-errors --recursive --sse S-ENCRYPT --storage-class S-CLAZZ "$name" "$s3path/$name" else /foo/bin/aws s3 cp --only-show-errors --sse S-ENCRYPT --storage-class S-CLAZZ "$name" "$s3path/$name" @@ -221,7 +225,9 @@ class S3BashLibTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -257,7 +263,9 @@ class S3BashLibTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -290,7 +298,9 @@ class S3BashLibTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -326,7 +336,9 @@ class S3BashLibTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + /some/bin/aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then /some/bin/aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else /some/bin/aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -416,7 +428,9 @@ class S3BashLibTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -503,7 +517,9 @@ class S3BashLibTest extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" diff --git a/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy b/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy index e4609ac45d..b2089d9d0b 100644 --- a/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy +++ b/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy @@ -72,7 +72,9 @@ class BashWrapperBuilderWithS3Test extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name" @@ -188,7 +190,9 @@ class BashWrapperBuilderWithS3Test extends Specification { nxf_s3_upload() { local name=$1 local s3path=$2 - if [[ -d "$name" ]]; then + if [[ "$name" == - ]]; then + aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path" + elif [[ -d "$name" ]]; then aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name" else aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name"