Skip to content

Commit

Permalink
Fix Missing AWS SSE encryption for begin and exit task files #2452
Browse files Browse the repository at this point in the history
  • Loading branch information
@pditommaso
pditommaso committed Nov 20, 2021
1 parent ad5ad97 commit acb53ee
Show file tree
Hide file tree
Showing 4 changed files with 29 additions and 17 deletions.
6 changes: 2 additions & 4 deletions plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategy.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,7 @@ class AwsBatchFileCopyStrategy extends SimpleFileCopyStrategy {
*/
@Override
String touchFile( Path file ) {
final aws = opts.getAwsCli()
"echo start | $aws s3 cp --only-show-errors - s3:/${Escape.path(file)}"
"echo start | nxf_s3_upload - s3:/${Escape.path(file)}"
}

/**
Expand All @@ -162,8 +161,7 @@ class AwsBatchFileCopyStrategy extends SimpleFileCopyStrategy {
* {@inheritDoc}
*/
String exitFile( Path path ) {
final aws = opts.getAwsCli()
"| $aws s3 cp --only-show-errors - s3:/${Escape.path(path)} || true"
"| nxf_s3_upload - s3:/${Escape.path(path)} || true"
}

/**
Expand Down
12 changes: 7 additions & 5 deletions plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import nextflow.executor.BashFunLib
class S3BashLib extends BashFunLib<S3BashLib> {

private String storageClass = 'STANDARD'
private String encryptionEncryption = ''
private String storageEncryption = ''
private String debug = ''
private String cli = 'aws'
private String retryMode
Expand Down Expand Up @@ -58,7 +58,7 @@ class S3BashLib extends BashFunLib<S3BashLib> {

S3BashLib withStorageEncryption(String value) {
if( value )
this.encryptionEncryption = value ? "--sse $value " : ''
this.storageEncryption = value ? "--sse $value " : ''
return this
}

Expand All @@ -78,10 +78,12 @@ class S3BashLib extends BashFunLib<S3BashLib> {
nxf_s3_upload() {
local name=\$1
local s3path=\$2
if [[ -d "\$name" ]]; then
$cli s3 cp --only-show-errors --recursive $debug$encryptionEncryption--storage-class $storageClass "\$name" "\$s3path/\$name"
if [[ "\$name" == - ]]; then
$cli s3 cp --only-show-errors $debug$storageEncryption--storage-class $storageClass - "\$s3path"
elif [[ -d "\$name" ]]; then
$cli s3 cp --only-show-errors --recursive $debug$storageEncryption--storage-class $storageClass "\$name" "\$s3path/\$name"
else
$cli s3 cp --only-show-errors $debug$encryptionEncryption--storage-class $storageClass "\$name" "\$s3path/\$name"
$cli s3 cp --only-show-errors $debug$storageEncryption--storage-class $storageClass "\$name" "\$s3path/\$name"
fi
}
Expand Down
12 changes: 8 additions & 4 deletions plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ class AwsBatchFileCopyStrategyTest extends Specification {
def RUN = Paths.get('/some/data/.command.run')
def copy = new AwsBatchFileCopyStrategy(Mock(TaskBean), new AwsOptions())
expect:
copy.touchFile(RUN) == "echo start | aws s3 cp --only-show-errors - s3://some/data/.command.run"
copy.touchFile(RUN) == "echo start | nxf_s3_upload - s3://some/data/.command.run"
copy.copyFile("nobel_prize_results.gz",Paths.get("/some/data/nobel_prize_results.gz")) == "nxf_s3_upload nobel_prize_results.gz s3://some/data"
copy.exitFile(EXIT) == "| aws s3 cp --only-show-errors - s3://some/path/.exitcode || true"
copy.exitFile(EXIT) == "| nxf_s3_upload - s3://some/path/.exitcode || true"
copy.stageInputFile(FILE, 'foo.txt') == """
downloads+=("nxf_s3_download s3://some/data/nobel_prize_results.gz foo.txt")
"""
Expand Down Expand Up @@ -188,7 +188,9 @@ class AwsBatchFileCopyStrategyTest extends Specification {
nxf_s3_upload() {
local name=$1
local s3path=$2
if [[ -d "$name" ]]; then
if [[ "$name" == - ]]; then
aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path"
elif [[ -d "$name" ]]; then
aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name"
else
aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name"
Expand Down Expand Up @@ -275,7 +277,9 @@ class AwsBatchFileCopyStrategyTest extends Specification {
nxf_s3_upload() {
local name=$1
local s3path=$2
if [[ -d "$name" ]]; then
if [[ "$name" == - ]]; then
/foo/aws s3 cp --only-show-errors --sse AES256 --storage-class STANDARD_IA - "$s3path"
elif [[ -d "$name" ]]; then
/foo/aws s3 cp --only-show-errors --recursive --sse AES256 --storage-class STANDARD_IA "$name" "$s3path/$name"
else
/foo/aws s3 cp --only-show-errors --sse AES256 --storage-class STANDARD_IA "$name" "$s3path/$name"
Expand Down
16 changes: 12 additions & 4 deletions plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ class AwsBatchScriptLauncherTest extends Specification {
nxf_s3_upload() {
local name=$1
local s3path=$2
if [[ -d "$name" ]]; then
if [[ "$name" == - ]]; then
/conda/bin/aws --region eu-west-1 s3 cp --only-show-errors --storage-class STANDARD - "$s3path"
elif [[ -d "$name" ]]; then
/conda/bin/aws --region eu-west-1 s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name"
else
/conda/bin/aws --region eu-west-1 s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name"
Expand Down Expand Up @@ -292,7 +294,9 @@ class AwsBatchScriptLauncherTest extends Specification {
nxf_s3_upload() {
local name=$1
local s3path=$2
if [[ -d "$name" ]]; then
if [[ "$name" == - ]]; then
aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path"
elif [[ -d "$name" ]]; then
aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name"
else
aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name"
Expand Down Expand Up @@ -433,7 +437,9 @@ class AwsBatchScriptLauncherTest extends Specification {
nxf_s3_upload() {
local name=$1
local s3path=$2
if [[ -d "$name" ]]; then
if [[ "$name" == - ]]; then
aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path"
elif [[ -d "$name" ]]; then
aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name"
else
aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name"
Expand Down Expand Up @@ -548,7 +554,9 @@ class AwsBatchScriptLauncherTest extends Specification {
nxf_s3_upload() {
local name=$1
local s3path=$2
if [[ -d "$name" ]]; then
if [[ "$name" == - ]]; then
aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path"
elif [[ -d "$name" ]]; then
aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name"
else
aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name"
Expand Down

0 comments on commit acb53ee

Please sign in to comment.