Move Sanitizer in Main Task (#18916)

microsoft · Sep 5, 2023 · 9b73032 · 9b73032
1 parent d967cd6
commit 9b73032
Show file tree

Hide file tree

Showing 65 changed files with 379 additions and 289 deletions.
diff --git a/Tasks/AzureFileCopyV1/AzureFileCopy.ps1 b/Tasks/AzureFileCopyV1/AzureFileCopy.ps1
@@ -52,7 +52,6 @@ $azCopyLocation = [System.IO.Path]::GetDirectoryName($azCopyExeLocation)
 
 # Initialize Azure.
 Import-Module $PSScriptRoot\ps_modules\VstsAzureHelpers_
-Import-Module $PSScriptRoot\ps_modules\Sanitizer
 Initialize-Azure
 
 # Import the loc strings.
@@ -76,6 +75,19 @@ if ($enableDetailedLoggingString -ne "true")
 # Telemetry
 Import-Module $PSScriptRoot\ps_modules\TelemetryHelper
 
+# Sanitizer
+Import-Module $PSScriptRoot\ps_modules\Sanitizer
+$useSanitizerCall = Get-SanitizerCallStatus
+$useSanitizerActivate = Get-SanitizerActivateStatus
+
+if ($useSanitizerCall) {
+    $sanitizedArguments = Protect-ScriptArguments -InputArgs $additionalArguments -TaskName "AzureFileCopyV1"
+}
+
+if ($useSanitizerActivate) {
+    $additionalArguments = $sanitizedArguments -join " "
+}
+
 #### MAIN EXECUTION OF AZURE FILE COPY TASK BEGINS HERE ####
 try {
     try
@@ -190,7 +202,7 @@ try {
             -storageAccountName $storageAccount -containerName $containerName -containerSasToken $containerSasToken -blobStorageEndpoint $blobStorageEndpoint -targetPath $targetPath -azCopyLocation $azCopyLocation `
             -resourceGroupName $environmentName -azureVMResourcesProperties $azureVMResourcesProperties -azureVMsCredentials $azureVMsCredentials `
             -cleanTargetBeforeCopy $cleanTargetBeforeCopy -communicationProtocol $useHttpsProtocolOption -skipCACheckOption $skipCACheckOption `
-            -enableDetailedLoggingString $enableDetailedLoggingString -additionalArguments $additionalArguments -copyFilesInParallel $copyFilesInParallel -connectionType $connectionType
+            -enableDetailedLoggingString $enableDetailedLoggingString -additionalArguments $additionalArguments -copyFilesInParallel $copyFilesInParallel -connectionType $connectionType -useSanitizerActivate $useSanitizerActivate
     }
     catch
     {

diff --git a/Tasks/AzureFileCopyV1/AzureFileCopyJob.ps1 b/Tasks/AzureFileCopyV1/AzureFileCopyJob.ps1
@@ -14,7 +14,8 @@ param (
     [string]$httpProtocolOption,
     [string]$skipCACheckOption,
     [string]$enableDetailedLogging,
-    [string]$additionalArguments
+    [string]$additionalArguments,
+    [bool]$useSanitizerActivate = $false
     )
 
     Write-Verbose "fqdn = $fqdn"
@@ -51,14 +52,8 @@ param (
         $blobStorageURI = $blobStorageEndpoint+$containerName+"/"+$blobPrefix
     }
 
-    $useSanitizerCall = Get-SanitizerCallStatus
-    $useSanitizerActivate = Get-SanitizerActivateStatus
-
-    if ($useSanitizerCall) {
-        $sanitizedArguments = Protect-ScriptArguments -InputArgs $additionalArguments -TaskName "AzureFileCopyV1"
-    }
-
     if ($useSanitizerActivate) {
+        $sanitizedArguments = [regex]::Split($additionalArguments, ' (?=(?:[^"]|"[^"]*")*$)')
         Copy-ToAzureMachines -MachineDnsName $fqdn -StorageAccountName $storageAccount -ContainerName $containerName -SasToken $sasToken -DestinationPath $targetPath -Credential $credential -AzCopyLocation $azCopyLocation -AdditionalArguments $sanitizedArguments -BlobStorageURI $blobStorageURI -WinRMPort $winRMPort $cleanTargetPathOption $skipCACheckOption $httpProtocolOption $enableDetailedLoggingOption
     } else {
         [String]$copyToAzureMachinesBlockString = [string]::Empty

diff --git a/Tasks/AzureFileCopyV1/Utility.ps1 b/Tasks/AzureFileCopyV1/Utility.ps1
@@ -278,17 +278,6 @@ function Upload-FilesToAzureContainer
         {
             $blobStorageURI = $blobStorageEndpoint+$containerName+"/"+$blobPrefix
         }
-
-        $useSanitizerCall = Get-SanitizerCallStatus
-        $useSanitizerActivate = Get-SanitizerActivateStatus
-
-        if ($useSanitizerCall) {
-            $sanitizedAdditionalArguments = Protect-ScriptArguments -InputArgs $additionalArguments -TaskName "AzureFileCopyV1"
-        }
-
-        if ($useSanitizerActivate) {
-            $additionalArguments = $sanitizedAdditionalArguments
-        }
 
         Copy-FilesToAzureBlob -SourcePathLocation $sourcePath -StorageAccountName $storageAccountName -ContainerName $containerName -BlobPrefix $blobPrefix -StorageAccountKey $storageKey -AzCopyLocation $azCopyLocation -AdditionalArguments $additionalArguments -BlobStorageURI $blobStorageURI
     }
@@ -1012,7 +1001,8 @@ function Copy-FilesSequentiallyToAzureVMs
           [string]$skipCACheckOption,
           [string][Parameter(Mandatory=$true)]$enableDetailedLoggingString,
           [string]$additionalArguments,
-          [string][Parameter(Mandatory=$true)]$connectionType)
+          [string][Parameter(Mandatory=$true)]$connectionType,
+          [string][Parameter(Mandatory=$false)]$useSanitizerActivate = $false)
 
     foreach ($resource in $azureVMResourcesProperties.Keys)
     {
@@ -1026,7 +1016,7 @@ function Copy-FilesSequentiallyToAzureVMs
         $deploymentUtilitiesLocation = Get-DeploymentModulePath
         $copyResponse = Invoke-Command -ScriptBlock $AzureFileCopyJob -ArgumentList `
                             $deploymentutilitieslocation, $resourceFQDN, $storageAccountName, $containerName, $containerSasToken, $blobStorageEndpoint, $azCopyLocation, $targetPath, $azureVMsCredentials, `
-                            $cleanTargetBeforeCopy, $resourceWinRMHttpsPort, $communicationProtocol, $skipCACheckOption, $enableDetailedLoggingString, $additionalArguments
+                            $cleanTargetBeforeCopy, $resourceWinRMHttpsPort, $communicationProtocol, $skipCACheckOption, $enableDetailedLoggingString, $additionalArguments, $useSanitizerActivate
 
         $status = $copyResponse.Status
 
@@ -1066,7 +1056,8 @@ function Copy-FilesParallellyToAzureVMs
           [string]$skipCACheckOption,
           [string][Parameter(Mandatory=$true)]$enableDetailedLoggingString,
           [string]$additionalArguments,
-          [string][Parameter(Mandatory=$true)]$connectionType)
+          [string][Parameter(Mandatory=$true)]$connectionType,
+          [string][Parameter(Mandatory=$false)]$useSanitizerActivate = $false)
 
     [hashtable]$Jobs = @{}
     $dtlsdkErrors = @()
@@ -1082,7 +1073,7 @@ function Copy-FilesParallellyToAzureVMs
 
         $job = Start-Job -ScriptBlock $AzureFileCopyJob -ArgumentList `
                    $deploymentutilitieslocation, $resourceFQDN, $storageAccountName, $containerName, $containerSasToken, $blobStorageEndpoint, $azCopyLocation, $targetPath, $azureVmsCredentials, `
-                   $cleanTargetBeforeCopy, $resourceWinRMHttpsPort, $communicationProtocol, $skipCACheckOption, $enableDetailedLoggingString, $additionalArguments
+                   $cleanTargetBeforeCopy, $resourceWinRMHttpsPort, $communicationProtocol, $skipCACheckOption, $enableDetailedLoggingString, $additionalArguments, $useSanitizerActivate
 
         $Jobs.Add($job.Id, $resourceProperties)
     }
@@ -1155,7 +1146,8 @@ function Copy-FilesToAzureVMsFromStorageContainer
           [string][Parameter(Mandatory=$true)]$enableDetailedLoggingString,
           [string]$additionalArguments,
           [string][Parameter(Mandatory=$true)]$copyFilesInParallel,
-          [string][Parameter(Mandatory=$true)]$connectionType)
+          [string][Parameter(Mandatory=$true)]$connectionType,
+          [string][Parameter(Mandatory=$false)]$useSanitizerActivate = $false)
 
     # copies files sequentially
     if ($copyFilesInParallel -eq "false" -or ( $azureVMResourcesProperties.Count -eq 1 ))
@@ -1166,7 +1158,7 @@ function Copy-FilesToAzureVMsFromStorageContainer
                 -blobStorageEndpoint $blobStorageEndpoint -targetPath $targetPath -azCopyLocation $azCopyLocation `
                 -azureVMResourcesProperties $azureVMResourcesProperties -azureVMsCredentials $azureVMsCredentials `
                 -cleanTargetBeforeCopy $cleanTargetBeforeCopy -communicationProtocol $communicationProtocol -skipCACheckOption $skipCACheckOption `
-                -enableDetailedLoggingString $enableDetailedLoggingString -additionalArguments $additionalArguments -connectionType $connectionType
+                -enableDetailedLoggingString $enableDetailedLoggingString -additionalArguments $additionalArguments -connectionType $connectionType -useSanitizerActivate $useSanitizerActivate
     }
     # copies files parallelly
     else
@@ -1176,7 +1168,7 @@ function Copy-FilesToAzureVMsFromStorageContainer
                 -blobStorageEndpoint $blobStorageEndpoint -targetPath $targetPath -azCopyLocation $azCopyLocation `
                 -azureVMResourcesProperties $azureVMResourcesProperties -azureVMsCredentials $azureVMsCredentials `
                 -cleanTargetBeforeCopy $cleanTargetBeforeCopy -communicationProtocol $communicationProtocol -skipCACheckOption $skipCACheckOption `
-                -enableDetailedLoggingString $enableDetailedLoggingString -additionalArguments $additionalArguments -connectionType $connectionType
+                -enableDetailedLoggingString $enableDetailedLoggingString -additionalArguments $additionalArguments -connectionType $connectionType -useSanitizerActivate $useSanitizerActivate
     }
 
     # if no error thrown, copy successfully succeeded

diff --git a/Tasks/AzureFileCopyV1/task.json b/Tasks/AzureFileCopyV1/task.json
@@ -13,8 +13,8 @@
     "author": "Microsoft Corporation",
     "version": {
         "Major": 1,
-        "Minor": 227,
-        "Patch": 1
+        "Minor": 228,
+        "Patch": 0
     },
     "demands": [
         "azureps"

diff --git a/Tasks/AzureFileCopyV1/task.loc.json b/Tasks/AzureFileCopyV1/task.loc.json
@@ -13,8 +13,8 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 1,
-    "Minor": 227,
-    "Patch": 1
+    "Minor": 228,
+    "Patch": 0
   },
   "demands": [
     "azureps"

diff --git a/Tasks/AzureFileCopyV2/AzureFileCopy.ps1 b/Tasks/AzureFileCopyV2/AzureFileCopy.ps1
@@ -65,7 +65,6 @@ Import-Module $PSScriptRoot\ps_modules\RemoteDeployer
 
 # Initialize Azure.
 Import-Module $PSScriptRoot\ps_modules\VstsAzureHelpers_
-Import-Module $PSScriptRoot\ps_modules\Sanitizer
 
 . "$PSScriptRoot\Utility.ps1"
 $endpoint = Get-Endpoint -connectedServiceName $connectedServiceName
@@ -84,6 +83,21 @@ $enableDetailedLogging = ($env:system_debug -eq "true")
 # Telemetry
 Import-Module $PSScriptRoot\ps_modules\TelemetryHelper
 
+# Sanitizer
+Import-Module $PSScriptRoot\ps_modules\Sanitizer
+$useSanitizerCall = Get-SanitizerCallStatus
+$useSanitizerActivate = Get-SanitizerActivateStatus
+
+if ($useSanitizerCall) {
+    $sanitizedArgumentsForBlobCopy = Protect-ScriptArguments -InputArgs $additionalArgumentsForBlobCopy -TaskName "AzureFileCopyV2"
+    $sanitizedArgumentsForVMCopy = Protect-ScriptArguments -InputArgs $additionalArgumentsForVMCopy -TaskName "AzureFileCopyV2"
+}
+
+if ($useSanitizerActivate) {
+    $additionalArgumentsForBlobCopy = $sanitizedArgumentsForBlobCopy -join " "
+    $additionalArgumentsForVMCopy = $sanitizedArgumentsForVMCopy -join " "
+}
+
 #### MAIN EXECUTION OF AZURE FILE COPY TASK BEGINS HERE ####
 try {
     try
@@ -183,7 +197,8 @@ try {
                                 -additionalArguments $additionalArgumentsForBlobCopy `
                                 -destinationType $destination `
                                 -useDefaultArguments $useDefaultArgumentsForBlobCopy `
-                                -azCopyLogFilePath $logFilePath
+                                -azCopyLogFilePath $logFilePath `
+                                -useSanitizerActivate $useSanitizerActivate
 
     # Complete the task if destination is azure blob
     if ($destination -eq "AzureBlob")
@@ -243,7 +258,8 @@ try {
                                                 -additionalArguments $additionalArgumentsForVMCopy `
                                                 -azCopyToolLocation $azCopyLocation `
                                                 -fileCopyJobScript $AzureFileCopyRemoteJob `
-                                                -enableDetailedLogging $enableDetailedLogging
+                                                -enableDetailedLogging $enableDetailedLogging `
+                                                -useSanitizerActivate $useSanitizerActivate
 
         Write-Output (Get-VstsLocString -Key "AFC_CopySuccessful" -ArgumentList $sourcePath, $environmentName)
     }

diff --git a/Tasks/AzureFileCopyV2/AzureFileCopyRemoteJob.ps1 b/Tasks/AzureFileCopyV2/AzureFileCopyRemoteJob.ps1
@@ -7,7 +7,8 @@ $AzureFileCopyRemoteJob = {
         [string]$azCopyToolFileNamesString,
         [string]$azCopyToolFileContentsString,
         [switch]$CleanTargetBeforeCopy,
-        [switch]$EnableDetailedLogging
+        [switch]$EnableDetailedLogging,
+        [bool]$useSanitizerActivate = $false
     )
 
     function Write-DetailLogs
@@ -118,14 +119,8 @@ $AzureFileCopyRemoteJob = {
             $additionalArguments = "/Z:`"$azCopyDestinationPath`" /V:`"$logFilePath`" /S /Y"
         }
 
-        $useSanitizerCall = Get-SanitizerCallStatus
-        $useSanitizerActivate = Get-SanitizerActivateStatus
-
-        if ($useSanitizerCall) {
-            $sanitizedArguments = Protect-ScriptArguments -InputArgs $additionalArguments -TaskName "AzureFileCopyV2"
-        }
-
         if ($useSanitizerActivate) {
+            $sanitizedArguments = [regex]::Split($additionalArguments, ' (?=(?:[^"]|"[^"]*")*$)')
             Write-DetailLogs "##[command] & `"$azCopyExeLocation`" /Source:`"$containerURL`" /Dest:`"$targetPath`" /SourceSAS:`"*****`" $sanitizedArguments"
             & $azCopyExeLocation /Source:$containerURL /Dest:$targetPath /SourceSAS:$containerSasToken $sanitizedArguments
         } else {

diff --git a/Tasks/AzureFileCopyV2/Utility.ps1 b/Tasks/AzureFileCopyV2/Utility.ps1
@@ -244,7 +244,8 @@ function Upload-FilesToAzureContainer
           [string]$additionalArguments,
           [string][Parameter(Mandatory=$true)]$destinationType,
           [bool]$useDefaultArguments,
-          [string]$azCopyLogFilePath
+          [string]$azCopyLogFilePath,
+          [bool]$useSanitizerActivate = $false
     )
 
     try
@@ -277,14 +278,8 @@ function Upload-FilesToAzureContainer
             )
         }
 
-        $useSanitizerCall = Get-SanitizerCallStatus
-        $useSanitizerActivate = Get-SanitizerActivateStatus
-
-        if ($useSanitizerCall) {
-            $sanitizedArguments = Protect-ScriptArguments -InputArgs $additionalArguments -TaskName "AzureFileCopyV2"
-        }
-
         if ($useSanitizerActivate) {
+            $sanitizedArguments = [regex]::Split($additionalArguments, ' (?=(?:[^"]|"[^"]*")*$)')
             Write-Output "##[command] & `"$azCopyExeLocation`" /Source:`"$resolvedSourcePath`" /Dest:`"$containerURL`" /@:`"$responseFile`" $sanitizedArguments"
             & $azCopyExeLocation /Source:$resolvedSourcePath /Dest:$containerURL /@:$responseFile $sanitizedArguments
         } else {
@@ -1099,7 +1094,8 @@ function Copy-FilesToAzureVMsFromStorageContainer
         [string]$additionalArguments,
         [string]$azCopyToolLocation,
         [scriptblock]$fileCopyJobScript,
-        [bool]$enableDetailedLogging
+        [bool]$enableDetailedLogging,
+        [bool]$useSanitizerActivate = $false
     )
 
     # Generate storage container URL
@@ -1119,7 +1115,7 @@ function Copy-FilesToAzureVMsFromStorageContainer
     $azCopyToolFileContentsString = $azCopyToolFileContents -join ";"
 
     # script block arguments
-    $scriptBlockArgs = " -containerURL '$containerURL' -targetPath '$targetPath' -containerSasToken '$containerSasToken' -additionalArguments '$additionalArguments' -azCopyToolFileNamesString '$azCopyToolFileNamesString' -azCopyToolFileContentsString '$azCopyToolFileContentsString'"
+    $scriptBlockArgs = " -containerURL '$containerURL' -targetPath '$targetPath' -containerSasToken '$containerSasToken' -additionalArguments '$additionalArguments' -azCopyToolFileNamesString '$azCopyToolFileNamesString' -azCopyToolFileContentsString '$azCopyToolFileContentsString' -useSanitizerActivate $useSanitizerActivate"
     if($cleanTargetBeforeCopy)
     {
         $scriptBlockArgs += " -CleanTargetBeforeCopy"

diff --git a/Tasks/AzureFileCopyV2/task.json b/Tasks/AzureFileCopyV2/task.json
@@ -13,8 +13,8 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 2,
-    "Minor": 227,
-    "Patch": 5
+    "Minor": 228,
+    "Patch": 1
   },
   "demands": [
     "azureps"

diff --git a/Tasks/AzureFileCopyV2/task.loc.json b/Tasks/AzureFileCopyV2/task.loc.json
@@ -13,8 +13,8 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 2,
-    "Minor": 227,
-    "Patch": 5
+    "Minor": 228,
+    "Patch": 1
   },
   "demands": [
     "azureps"

diff --git a/Tasks/AzureFileCopyV3/AzureFileCopy.ps1 b/Tasks/AzureFileCopyV3/AzureFileCopy.ps1
@@ -62,7 +62,6 @@ Import-Module $PSScriptRoot\ps_modules\RemoteDeployer
 
 # Initialize Azure.
 Import-Module $PSScriptRoot\ps_modules\VstsAzureHelpers_
-Import-Module $PSScriptRoot\ps_modules\Sanitizer
 
 $endpoint = Get-VstsEndpoint -Name $connectedServiceName -Require
 $vstsEndpoint = Get-VstsEndpoint -Name SystemVssConnection -Require
@@ -90,6 +89,21 @@ $enableDetailedLogging = ($env:system_debug -eq "true")
 # Telemetry
 Import-Module $PSScriptRoot\ps_modules\TelemetryHelper
 
+# Sanitizer
+Import-Module $PSScriptRoot\ps_modules\Sanitizer
+$useSanitizerCall = Get-SanitizerCallStatus
+$useSanitizerActivate = Get-SanitizerActivateStatus
+
+if ($useSanitizerCall) {
+    $sanitizedArgumentsForBlobCopy = Protect-ScriptArguments -InputArgs $additionalArgumentsForBlobCopy -TaskName "AzureFileCopyV3"
+    $sanitizedArgumentsForVMCopy = Protect-ScriptArguments -InputArgs $additionalArgumentsForVMCopy -TaskName "AzureFileCopyV3"
+}
+
+if ($useSanitizerActivate) {
+    $additionalArgumentsForBlobCopy = $sanitizedArgumentsForBlobCopy -join " "
+    $additionalArgumentsForVMCopy = $sanitizedArgumentsForVMCopy -join " "
+}
+
 #### MAIN EXECUTION OF AZURE FILE COPY TASK BEGINS HERE ####
 try {
     try
@@ -183,7 +197,8 @@ try {
                                 -additionalArguments $additionalArgumentsForBlobCopy `
                                 -destinationType $destination `
                                 -useDefaultArguments $useDefaultArgumentsForBlobCopy `
-                                -azCopyLogFilePath $logFilePath
+                                -azCopyLogFilePath $logFilePath `
+                                -useSanitizerActivate $useSanitizerActivate
 
     # Complete the task if destination is azure blob
     if ($destination -eq "AzureBlob")
@@ -243,7 +258,8 @@ try {
                                                 -additionalArguments $additionalArgumentsForVMCopy `
                                                 -azCopyToolLocation $azCopyLocation `
                                                 -fileCopyJobScript $AzureFileCopyRemoteJob `
-                                                -enableDetailedLogging $enableDetailedLogging
+                                                -enableDetailedLogging $enableDetailedLogging `
+                                                -useSanitizerActivate $useSanitizerActivate
 
         Write-Output (Get-VstsLocString -Key "AFC_CopySuccessful" -ArgumentList $sourcePath, $environmentName)
     }

diff --git a/Tasks/AzureFileCopyV3/AzureFileCopyRemoteJob.ps1 b/Tasks/AzureFileCopyV3/AzureFileCopyRemoteJob.ps1
@@ -7,7 +7,8 @@ $AzureFileCopyRemoteJob = {
         [string]$azCopyToolFileNamesString,
         [string]$azCopyToolFileContentsString,
         [switch]$CleanTargetBeforeCopy,
-        [switch]$EnableDetailedLogging
+        [switch]$EnableDetailedLogging,
+        [bool]$useSanitizerActivate = $false
     )
 
     function Write-DetailLogs
@@ -118,14 +119,8 @@ $AzureFileCopyRemoteJob = {
             $additionalArguments = "/Z:`"$azCopyDestinationPath`" /V:`"$logFilePath`" /S /Y"
         }
 
-        $useSanitizerCall = Get-SanitizerCallStatus
-        $useSanitizerActivate = Get-SanitizerActivateStatus
-
-        if ($useSanitizerCall){
-            $sanitizedArguments = Protect-ScriptArguments -InputArgs $additionalArguments -TaskName "AzureFileCopyV3"
-        }
-
         if ($useSanitizerActivate) {
+            $sanitizedArguments = [regex]::Split($additionalArguments, ' (?=(?:[^"]|"[^"]*")*$)')
             Write-DetailLogs "##[command] & `"$azCopyExeLocation`" /Source:`"$containerURL`" /Dest:`"$targetPath`" /SourceSAS:`"*****`" $sanitizedArguments"
             & $azCopyExeLocation /Source:$containerURL /Dest:$targetPath /SourceSAS:$containerSasToken $sanitizedArguments
         } else {