Enhance the azcopy options configuration scope

docs/azure.rst

@@ -392,6 +392,9 @@ Name                                           Description
 azure.activeDirectory.servicePrincipalId        The service principal client ID
 azure.activeDirectory.servicePrincipalSecret    The service principal client secret
 azure.activeDirectory.tenantId                  The Azure tenant ID
+azure.azcopy.blockSize                          The block size (in MiB) when uploading to Azure Storage, and downloading from Azure Storage (default: 4 , requires ``[email protected]``).
+azure.azcopy.blobTier                           The blob tier to be used while uploading to Azure Storage (default: 'None' , requires ``[email protected]``).
+azure.azcopy.overwrite                          Overwrite the conflicting files and blobs at the destination if this flag is set to true. (default: "false")
 azure.storage.accountName                       The blob storage account name
 azure.storage.accountKey                        The blob storage account key
 azure.storage.sasToken                          The blob storage shared access signature token. This can be provided as an alternative to the ``accountKey`` setting.
@@ -404,7 +407,7 @@ azure.batch.autoPoolMode                        Enable the automatic creation of
 azure.batch.allowPoolCreation                   Enable the automatic creation of batch pools specified in the Nextflow configuration file (default: ``false``).
 azure.batch.deleteJobsOnCompletion              Enable the automatic deletion of jobs created by the pipeline execution (default: ``true``).
 azure.batch.deletePoolsOnCompletion             Enable the automatic deletion of compute node pools upon pipeline completion (default: ``false``).
-azure.batch.copyToolInstallMode                 Specify where the `azcopy` tool used by Nextflow. When ``node`` is specified it's copied once during the pool creation. When ``task`` is provider, it's installed for each task execution (default: ``node``).
+azure.batch.copyToolInstallMode                 Specify where the ``azcopy`` tool used by Nextflow. When ``node`` is specified it's copied once during the pool creation. When ``task`` is provider, it's installed for each task execution (default: ``node``).
 azure.batch.pools.<name>.publisher              Specify the publisher of virtual machine type used by the pool identified with ``<name>`` (default: ``microsoft-azure-batch``, requires ``[email protected]``).
 azure.batch.pools.<name>.offer                  Specify the offer type of the virtual machine type used by the pool identified with ``<name>`` (default: ``centos-container``, requires ``[email protected]``).
 azure.batch.pools.<name>.sku                    Specify the ID of the Compute Node agent SKU which the pool identified with ``<name>`` supports (default: ``batch.node.centos 8``, requires ``[email protected]``).

plugins/nf-azure/build.gradle

@@ -24,7 +24,7 @@ sourceSets {
     main.resources.srcDirs = ['src/resources']
     test.groovy.srcDirs = ['src/test']
     test.java.srcDirs = ['src/testResources']
-    test.resources.srcDirs = []
+    test.resources.srcDirs = ['src/testResources']
 }
 
 configurations {

plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchExecutor.groovy

@@ -52,7 +52,7 @@ class AzBatchExecutor extends Executor implements ExtensionPoint {
     private AzBatchService batchService
 
     /**
-     * @return {@code true} to signal containers are managed directly the AWS Batch service
+     * @return {@code true} to signal containers are managed directly the AZ Batch service
      */
     final boolean isContainerNative() {
         return true
@@ -70,7 +70,7 @@ class AzBatchExecutor extends Executor implements ExtensionPoint {
 
     protected void validateWorkDir() {
         /*
-         * make sure the work dir is an Azure bucket
+         * make sure the work dir is an AZ bucket
          */
         if( !(workDir instanceof AzPath) ) {
             session.abort()

plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy

@@ -669,7 +669,7 @@ class AzBatchService implements Closeable {
                 .withVirtualMachineConfiguration(poolVmConfig(spec.opts))
                 // https://docs.microsoft.com/en-us/azure/batch/batch-pool-vm-sizes
                 .withVmSize(spec.vmType.name)
-                // same as the num ofd cores
+                // same as the num of cores
                 // https://docs.microsoft.com/en-us/azure/batch/batch-parallel-node-tasks
                 .withTaskSlotsPerNode(spec.vmType.numberOfCores)
                 .withStartTask(poolStartTask)

plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzCopyOpts.groovy

@@ -1,7 +1,7 @@
 /*
  * Copyright 2021, Microsoft Corp
  * Copyright 2022, Seqera Labs
- * 
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -26,22 +26,37 @@ import groovy.transform.CompileStatic
 @CompileStatic
 class AzCopyOpts {
 
-    static public final String DEFAULT_BLOCK_SIZE = "4"
-    static public final String DEFAULT_BLOB_TIER = "None"
+    //-----------------------------------------------------
+    // Default values for azcopy copy command options
+    //-----------------------------------------------------
 
+    //Use this block size (specified in MiB) when uploading/downloading to/from Azure Storage (azcopy default: 8)
+    //Can be in decimals for eg. 0.25 and maximum value is 100. https://github.com/Azure/azure-storage-azcopy/wiki/Cost-of-data-transfers
+    static public final String DEFAULT_BLOCK_SIZE = "4"
     String blockSize
+
+    //Upload block blob to Azure Storage using this blob tier. (azcopy default: "None")
+    static public final String DEFAULT_BLOB_TIER = "None" // hot (None) | cool | archive
     String blobTier
 
+    //Overwrite the conflicting files and blobs at the destination if this flag is set to true. (azcopy default: true)
+    static public final String DEFAULT_OVERWRITE = "false" // true | false | prompt | ifSourceNewer
+    String overwrite
+
     AzCopyOpts() {
         this.blockSize = DEFAULT_BLOCK_SIZE
-        this.blobTier =  DEFAULT_BLOB_TIER
+        this.blobTier = DEFAULT_BLOB_TIER
+        this.overwrite = DEFAULT_OVERWRITE
     }
 
 
     AzCopyOpts(Map config) {
-        assert config!=null
+        assert config != null
+
         this.blockSize = config.blockSize ?: DEFAULT_BLOCK_SIZE
         this.blobTier = config.blobTier ?: DEFAULT_BLOB_TIER
+        this.overwrite = config.overwrite ?: DEFAULT_OVERWRITE
+
     }
 
 }

plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy

@@ -31,30 +31,38 @@ class AzBashLib extends BashFunLib<AzBashLib> {
 
     private String blockSize = AzCopyOpts.DEFAULT_BLOCK_SIZE
     private String blobTier = AzCopyOpts.DEFAULT_BLOB_TIER
-
+    private String overwrite = AzCopyOpts.DEFAULT_OVERWRITE
 
     AzBashLib withBlockSize(String value) {
-        if( value )
+        if (value)
             this.blockSize = value
         return this
     }
 
     AzBashLib withBlobTier(String value) {
-        if( value )
+        if (value)
             this.blobTier = value
         return this
     }
 
+    AzBashLib withOverwrite(String value) {
+        if (value)
+            this.overwrite = value
+        return this
+    }
 
+    // Custom env variables are prefixed with underscore(_)
     protected String setupAzCopyOpts() {
         """
-        # custom env variables used for azcopy opts
-        export AZCOPY_BLOCK_SIZE_MB=${blockSize}
-        export AZCOPY_BLOCK_BLOB_TIER=${blobTier}
+        # Env variables used for azcopy opts
+        export _AZCOPY_BLOCK_SIZE_MB=${blockSize}
+        export _AZCOPY_BLOCK_BLOB_TIER=${blobTier}
+        export _AZCOPY_PUT_MD5="--put-md5"
+        export _AZCOPY_CHECK_MD5="FailIfDifferent"
+        export _AZCOPY_OVERWRITE=${overwrite}
         """.stripIndent()
     }
 
-
     protected String azLib() {
         '''
         nxf_az_upload() {
@@ -65,12 +73,12 @@ class AzBashLib extends BashFunLib<AzBashLib> {
 
             if [[ -d $name ]]; then
               if [[ "$base_name" == "$name" ]]; then
-                azcopy cp "$name" "$target?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
+                azcopy cp "$name" "$target?$AZ_SAS" --recursive --block-blob-tier $_AZCOPY_BLOCK_BLOB_TIER --block-size-mb $_AZCOPY_BLOCK_SIZE_MB --overwrite $_AZCOPY_OVERWRITE $_AZCOPY_PUT_MD5
               else
-                azcopy cp "$name" "$target/$dir_name?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
+                azcopy cp "$name" "$target/$dir_name?$AZ_SAS" --recursive --block-blob-tier $_AZCOPY_BLOCK_BLOB_TIER --block-size-mb $_AZCOPY_BLOCK_SIZE_MB --overwrite $_AZCOPY_OVERWRITE $_AZCOPY_PUT_MD5
               fi
             else
-              azcopy cp "$name" "$target/$name?$AZ_SAS" --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
+              azcopy cp "$name" "$target/$name?$AZ_SAS" --block-blob-tier $_AZCOPY_BLOCK_BLOB_TIER --block-size-mb $_AZCOPY_BLOCK_SIZE_MB --overwrite $_AZCOPY_OVERWRITE $_AZCOPY_PUT_MD5
             fi
         }
 
@@ -84,7 +92,7 @@ class AzBashLib extends BashFunLib<AzBashLib> {
             ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
                 ## if fails check if it was trying to download a directory
                 mkdir -p $target
-                azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
+                azcopy cp "$source/*?$AZ_SAS" "$target" --recursive --check-md5 $_AZCOPY_CHECK_MD5 --overwrite $_AZCOPY_OVERWRITE >/dev/null || {
                     rm -rf $target
                     >&2 echo "Unable to download path: $source"
                     exit 1
@@ -107,6 +115,7 @@ class AzBashLib extends BashFunLib<AzBashLib> {
                 .withDelayBetweenAttempts(delayBetweenAttempts)
                 .withBlobTier(opts.blobTier)
                 .withBlockSize(opts.blockSize)
+                .withOverwrite(opts.overwrite)
                 .render()
     }
 

plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzCopyOptsTest.groovy

@@ -8,20 +8,23 @@ import spock.lang.Specification
  */
 class AzCopyOptsTest extends Specification {
 
-    def 'should get block size and blob tier'() {
+    def 'should get the default opts'() {
         when:
         def opts1 = new AzCopyOpts([:])
         then:
         opts1.blobTier == 'None'
         opts1.blockSize == '4'
+        opts1.overwrite == "false"
+    }
 
+    def 'should get the custom opts'() {
         when:
-        def opts2 = new AzCopyOpts(
-                [blobTier: 'Hot', blockSize: '100'])
+        def opts2 = new AzCopyOpts([blobTier : 'Cool',
+                                    blockSize: '100',
+                                    overwrite: "true"])
         then:
-        opts2.blobTier == 'Hot'
+        opts2.blobTier == 'Cool'
         opts2.blockSize == '100'
-
+        opts2.overwrite == "true"
     }
-
 }