Skip to content

Commit

Permalink
Sanitizer Additional Supports (#18940)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ozanMSFT
ozanMSFT authored Sep 11, 2023
1 parent 2a608dd commit 61960ba
Show file tree
Hide file tree
Showing 44 changed files with 129 additions and 79 deletions.
15 changes: 11 additions & 4 deletions Tasks/AzureFileCopyV1/AzureFileCopyJob.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ param (
[string]$skipCACheckOption,
[string]$enableDetailedLogging,
[string]$additionalArguments,
[bool]$useSanitizerActivate = $false
[string]$useSanitizerActivate
)

Write-Verbose "fqdn = $fqdn"
Expand Down Expand Up @@ -52,9 +52,16 @@ param (
$blobStorageURI = $blobStorageEndpoint+$containerName+"/"+$blobPrefix
}

if ($useSanitizerActivate) {
$sanitizedArguments = [regex]::Split($additionalArguments, ' (?=(?:[^"]|"[^"]*")*$)')
Copy-ToAzureMachines -MachineDnsName $fqdn -StorageAccountName $storageAccount -ContainerName $containerName -SasToken $sasToken -DestinationPath $targetPath -Credential $credential -AzCopyLocation $azCopyLocation -AdditionalArguments $sanitizedArguments -BlobStorageURI $blobStorageURI -WinRMPort $winRMPort $cleanTargetPathOption $skipCACheckOption $httpProtocolOption $enableDetailedLoggingOption
if (($useSanitizerActivate -eq "true") -and (-not [string]::IsNullOrWhiteSpace($additionalArguments))) {
# not splitting $additionalArguments, since copy-toazuremachines expects it as a single string
# TODO: disabled temporarily direct call due to the positional error
#Copy-ToAzureMachines -MachineDnsName $fqdn -StorageAccountName $storageAccount -ContainerName $containerName -SasToken $sasToken -DestinationPath $targetPath -Credential $credential -AzCopyLocation $azCopyLocation -AdditionalArguments $additionalArguments -BlobStorageURI $blobStorageURI -WinRMPort $winRMPort $cleanTargetPathOption $skipCACheckOption $httpProtocolOption $enableDetailedLoggingOption

# TODO: temporarily using old invoke
$copyToAzureMachinesBlockString = "Copy-ToAzureMachines -MachineDnsName `$fqdn -StorageAccountName `$storageAccount -ContainerName `$containerName -SasToken `$sasToken -DestinationPath `$targetPath -Credential `$credential -AzCopyLocation `$azCopyLocation -AdditionalArguments `$additionalArguments -BlobStorageURI `$blobStorageURI -WinRMPort $winRMPort $cleanTargetPathOption $skipCACheckOption $httpProtocolOption $enableDetailedLoggingOption"
[scriptblock]$copyToAzureMachinesBlock = [scriptblock]::Create($copyToAzureMachinesBlockString)
$copyResponse = Invoke-Command -ScriptBlock $copyToAzureMachinesBlock
Write-Output $copyResponse
} else {
[String]$copyToAzureMachinesBlockString = [string]::Empty
if([string]::IsNullOrWhiteSpace($additionalArguments))
Expand Down
6 changes: 3 additions & 3 deletions Tasks/AzureFileCopyV1/Utility.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1002,7 +1002,7 @@ function Copy-FilesSequentiallyToAzureVMs
[string][Parameter(Mandatory=$true)]$enableDetailedLoggingString,
[string]$additionalArguments,
[string][Parameter(Mandatory=$true)]$connectionType,
[string][Parameter(Mandatory=$false)]$useSanitizerActivate = $false)
[string][Parameter(Mandatory=$false)]$useSanitizerActivate)

foreach ($resource in $azureVMResourcesProperties.Keys)
{
Expand Down Expand Up @@ -1057,7 +1057,7 @@ function Copy-FilesParallellyToAzureVMs
[string][Parameter(Mandatory=$true)]$enableDetailedLoggingString,
[string]$additionalArguments,
[string][Parameter(Mandatory=$true)]$connectionType,
[string][Parameter(Mandatory=$false)]$useSanitizerActivate = $false)
[string][Parameter(Mandatory=$false)]$useSanitizerActivate)

[hashtable]$Jobs = @{}
$dtlsdkErrors = @()
Expand Down Expand Up @@ -1147,7 +1147,7 @@ function Copy-FilesToAzureVMsFromStorageContainer
[string]$additionalArguments,
[string][Parameter(Mandatory=$true)]$copyFilesInParallel,
[string][Parameter(Mandatory=$true)]$connectionType,
[string][Parameter(Mandatory=$false)]$useSanitizerActivate = $false)
[string][Parameter(Mandatory=$false)]$useSanitizerActivate)

# copies files sequentially
if ($copyFilesInParallel -eq "false" -or ( $azureVMResourcesProperties.Count -eq 1 ))
Expand Down
2 changes: 1 addition & 1 deletion Tasks/AzureFileCopyV1/task.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 1,
"Minor": 228,
"Patch": 0
"Patch": 1
},
"demands": [
"azureps"
Expand Down
2 changes: 1 addition & 1 deletion Tasks/AzureFileCopyV1/task.loc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 1,
"Minor": 228,
"Patch": 0
"Patch": 1
},
"demands": [
"azureps"
Expand Down
5 changes: 3 additions & 2 deletions Tasks/AzureFileCopyV2/AzureFileCopyRemoteJob.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ $AzureFileCopyRemoteJob = {
[string]$azCopyToolFileContentsString,
[switch]$CleanTargetBeforeCopy,
[switch]$EnableDetailedLogging,
[bool]$useSanitizerActivate = $false
[switch]$useSanitizerActivate
)

function Write-DetailLogs
Expand Down Expand Up @@ -120,9 +120,10 @@ $AzureFileCopyRemoteJob = {
}

if ($useSanitizerActivate) {
# Splitting arguments on space, but not on space inside quotes
$sanitizedArguments = [regex]::Split($additionalArguments, ' (?=(?:[^"]|"[^"]*")*$)')
Write-DetailLogs "##[command] & `"$azCopyExeLocation`" /Source:`"$containerURL`" /Dest:`"$targetPath`" /SourceSAS:`"*****`" $sanitizedArguments"
& $azCopyExeLocation /Source:$containerURL /Dest:$targetPath /SourceSAS:$containerSasToken $sanitizedArguments
& "$azCopyExeLocation" /Source:"$containerURL" /Dest:"$targetPath" /SourceSAS:"$containerSasToken" $sanitizedArguments
} else {
Write-DetailLogs "##[command] & `"$azCopyExeLocation`" /Source:`"$containerURL`" /Dest:`"$targetPath`" /SourceSAS:`"*****`" $additionalArguments"
$azCopyCommand = "& `"$azCopyExeLocation`" /Source:`"$containerURL`" /Dest:`"$targetPath`" /SourceSAS:`"$containerSasToken`" $additionalArguments"
Expand Down
6 changes: 5 additions & 1 deletion Tasks/AzureFileCopyV2/Utility.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1115,7 +1115,7 @@ function Copy-FilesToAzureVMsFromStorageContainer
$azCopyToolFileContentsString = $azCopyToolFileContents -join ";"

# script block arguments
$scriptBlockArgs = " -containerURL '$containerURL' -targetPath '$targetPath' -containerSasToken '$containerSasToken' -additionalArguments '$additionalArguments' -azCopyToolFileNamesString '$azCopyToolFileNamesString' -azCopyToolFileContentsString '$azCopyToolFileContentsString' -useSanitizerActivate $useSanitizerActivate"
$scriptBlockArgs = " -containerURL '$containerURL' -targetPath '$targetPath' -containerSasToken '$containerSasToken' -additionalArguments '$additionalArguments' -azCopyToolFileNamesString '$azCopyToolFileNamesString' -azCopyToolFileContentsString '$azCopyToolFileContentsString'"
if($cleanTargetBeforeCopy)
{
$scriptBlockArgs += " -CleanTargetBeforeCopy"
Expand All @@ -1124,6 +1124,10 @@ function Copy-FilesToAzureVMsFromStorageContainer
{
$scriptBlockArgs += " -EnableDetailedLogging"
}
if($useSanitizerActivate)
{
$scriptBlockArgs += " -useSanitizerActivate"
}

$remoteScriptJobArguments = @{
inline = $true;
Expand Down
2 changes: 1 addition & 1 deletion Tasks/AzureFileCopyV2/task.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 2,
"Minor": 228,
"Patch": 1
"Patch": 3
},
"demands": [
"azureps"
Expand Down
2 changes: 1 addition & 1 deletion Tasks/AzureFileCopyV2/task.loc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 2,
"Minor": 228,
"Patch": 1
"Patch": 3
},
"demands": [
"azureps"
Expand Down
5 changes: 3 additions & 2 deletions Tasks/AzureFileCopyV3/AzureFileCopyRemoteJob.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ $AzureFileCopyRemoteJob = {
[string]$azCopyToolFileContentsString,
[switch]$CleanTargetBeforeCopy,
[switch]$EnableDetailedLogging,
[bool]$useSanitizerActivate = $false
[switch]$useSanitizerActivate
)

function Write-DetailLogs
Expand Down Expand Up @@ -120,9 +120,10 @@ $AzureFileCopyRemoteJob = {
}

if ($useSanitizerActivate) {
# Splitting arguments on space, but not on space inside quotes
$sanitizedArguments = [regex]::Split($additionalArguments, ' (?=(?:[^"]|"[^"]*")*$)')
Write-DetailLogs "##[command] & `"$azCopyExeLocation`" /Source:`"$containerURL`" /Dest:`"$targetPath`" /SourceSAS:`"*****`" $sanitizedArguments"
& $azCopyExeLocation /Source:$containerURL /Dest:$targetPath /SourceSAS:$containerSasToken $sanitizedArguments
& "$azCopyExeLocation" /Source:"$containerURL" /Dest:"$targetPath" /SourceSAS:"$containerSasToken" $sanitizedArguments
} else {
Write-DetailLogs "##[command] & `"$azCopyExeLocation`" /Source:`"$containerURL`" /Dest:`"$targetPath`" /SourceSAS:`"*****`" $additionalArguments"
$azCopyCommand = "& `"$azCopyExeLocation`" /Source:`"$containerURL`" /Dest:`"$targetPath`" /SourceSAS:`"$containerSasToken`" $additionalArguments"
Expand Down
6 changes: 5 additions & 1 deletion Tasks/AzureFileCopyV3/Utility.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -937,7 +937,7 @@ function Copy-FilesToAzureVMsFromStorageContainer
$azCopyToolFileContentsString = $azCopyToolFileContents -join ";"

# script block arguments
$scriptBlockArgs = " -containerURL '$containerURL' -targetPath '$targetPath' -containerSasToken '$containerSasToken' -additionalArguments '$additionalArguments' -azCopyToolFileNamesString '$azCopyToolFileNamesString' -azCopyToolFileContentsString '$azCopyToolFileContentsString' -useSanitizerActivate $useSanitizerActivate"
$scriptBlockArgs = " -containerURL '$containerURL' -targetPath '$targetPath' -containerSasToken '$containerSasToken' -additionalArguments '$additionalArguments' -azCopyToolFileNamesString '$azCopyToolFileNamesString' -azCopyToolFileContentsString '$azCopyToolFileContentsString'"
if($cleanTargetBeforeCopy)
{
$scriptBlockArgs += " -CleanTargetBeforeCopy"
Expand All @@ -946,6 +946,10 @@ function Copy-FilesToAzureVMsFromStorageContainer
{
$scriptBlockArgs += " -EnableDetailedLogging"
}
if($useSanitizerActivate)
{
$scriptBlockArgs += " -useSanitizerActivate"
}

$remoteScriptJobArguments = @{
inline = $true;
Expand Down
2 changes: 1 addition & 1 deletion Tasks/AzureFileCopyV3/task.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 3,
"Minor": 228,
"Patch": 1
"Patch": 3
},
"demands": [
"azureps"
Expand Down
2 changes: 1 addition & 1 deletion Tasks/AzureFileCopyV3/task.loc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 3,
"Minor": 228,
"Patch": 1
"Patch": 3
},
"demands": [
"azureps"
Expand Down
5 changes: 3 additions & 2 deletions Tasks/AzureFileCopyV4/AzureFileCopyRemoteJob.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ $AzureFileCopyRemoteJob = {
[string]$additionalArguments,
[switch]$CleanTargetBeforeCopy,
[switch]$EnableDetailedLogging,
[bool]$useSanitizerActivate = $false
[switch]$useSanitizerActivate
)

function Write-DetailLogs
Expand Down Expand Up @@ -99,9 +99,10 @@ $AzureFileCopyRemoteJob = {
}

if ($useSanitizerActivate) {
# Splitting arguments on space, but not on space inside quotes
$sanitizedArguments = [regex]::Split($additionalArguments, ' (?=(?:[^"]|"[^"]*")*$)')
Write-DetailLogs "##[command] & azcopy copy `"$containerURL*****`" `"$targetPath`" $sanitizedArguments"
& azcopy copy $targetPath $containerURL/*$containerSasToken $sanitizedArguments
& azcopy copy "$containerURL/*$containerSasToken" "$targetPath" $sanitizedArguments
} else {
Write-DetailLogs "##[command] & azcopy copy `"$containerURL*****`" `"$targetPath`" $additionalArguments"
$azCopyCommand = "& azcopy copy `"$containerURL/*$containerSasToken`" `"$targetPath`" $additionalArguments"
Expand Down
6 changes: 5 additions & 1 deletion Tasks/AzureFileCopyV4/Utility.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -988,7 +988,7 @@ function Copy-FilesToAzureVMsFromStorageContainer
}

# script block arguments
$scriptBlockArgs = " -containerURL '$containerURL' -targetPath '$targetPath' -containerSasToken '$containerSasToken' -additionalArguments '$additionalArguments' -useSanitizerActivate $useSanitizerActivate"
$scriptBlockArgs = " -containerURL '$containerURL' -targetPath '$targetPath' -containerSasToken '$containerSasToken' -additionalArguments '$additionalArguments'"
if($cleanTargetBeforeCopy)
{
$scriptBlockArgs += " -CleanTargetBeforeCopy"
Expand All @@ -997,6 +997,10 @@ function Copy-FilesToAzureVMsFromStorageContainer
{
$scriptBlockArgs += " -EnableDetailedLogging"
}
if($useSanitizerActivate)
{
$scriptBlockArgs += " -useSanitizerActivate"
}

$remoteScriptJobArguments = @{
inline = $true;
Expand Down
2 changes: 1 addition & 1 deletion Tasks/AzureFileCopyV4/task.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 4,
"Minor": 228,
"Patch": 0
"Patch": 1
},
"demands": [
"azureps"
Expand Down
2 changes: 1 addition & 1 deletion Tasks/AzureFileCopyV4/task.loc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 4,
"Minor": 228,
"Patch": 0
"Patch": 1
},
"demands": [
"azureps"
Expand Down
5 changes: 3 additions & 2 deletions Tasks/AzureFileCopyV5/AzureFileCopyRemoteJob.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ $AzureFileCopyRemoteJob = {
[string]$additionalArguments,
[switch]$CleanTargetBeforeCopy,
[switch]$EnableDetailedLogging,
[bool]$useSanitizerActivate = $false
[switch]$useSanitizerActivate
)

function Write-DetailLogs
Expand Down Expand Up @@ -99,9 +99,10 @@ $AzureFileCopyRemoteJob = {
}

if ($useSanitizerActivate) {
# Splitting arguments on space, but not on space inside quotes
$sanitizedArguments = [regex]::Split($additionalArguments, ' (?=(?:[^"]|"[^"]*")*$)')
Write-DetailLogs "##[command] & azcopy copy `"$containerURL*****`" `"$targetPath`" $sanitizedArguments"
& azcopy copy $targetPath $containerURL/*$containerSasToken $sanitizedArguments
& azcopy copy "$containerURL/*$containerSasToken" "$targetPath" $sanitizedArguments
} else {
Write-DetailLogs "##[command] & azcopy copy `"$containerURL*****`" `"$targetPath`" $additionalArguments"
$azCopyCommand = "& azcopy copy `"$containerURL/*$containerSasToken`" `"$targetPath`" $additionalArguments"
Expand Down
7 changes: 6 additions & 1 deletion Tasks/AzureFileCopyV5/Utility.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ function Upload-FilesToAzureContainer
}

if ($useSanitizerActivate) {
# Splitting arguments on space, but not on space inside quotes
$sanitizedArguments = [regex]::Split($additionalArguments, ' (?=(?:[^"]|"[^"]*")*$)')
Write-Output "##[command] & azcopy copy `"$sourcePath`" `"$containerURL`" $sanitizedArguments"
& azcopy copy $sourcePath $containerURL$containerSasToken $sanitizedArguments
Expand Down Expand Up @@ -1001,7 +1002,7 @@ function Copy-FilesToAzureVMsFromStorageContainer
}

# script block arguments
$scriptBlockArgs = " -containerURL '$containerURL' -targetPath '$targetPath' -containerSasToken '$containerSasToken' -additionalArguments '$additionalArguments' -useSanitizerActivate $useSanitizerActivate"
$scriptBlockArgs = " -containerURL '$containerURL' -targetPath '$targetPath' -containerSasToken '$containerSasToken' -additionalArguments '$additionalArguments'"
if($cleanTargetBeforeCopy)
{
$scriptBlockArgs += " -CleanTargetBeforeCopy"
Expand All @@ -1010,6 +1011,10 @@ function Copy-FilesToAzureVMsFromStorageContainer
{
$scriptBlockArgs += " -EnableDetailedLogging"
}
if($useSanitizerActivate)
{
$scriptBlockArgs += " -useSanitizerActivate"
}

$remoteScriptJobArguments = @{
inline = $true;
Expand Down
2 changes: 1 addition & 1 deletion Tasks/AzureFileCopyV5/task.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 5,
"Minor": 228,
"Patch": 0
"Patch": 1
},
"demands": [
"azureps"
Expand Down
2 changes: 1 addition & 1 deletion Tasks/AzureFileCopyV5/task.loc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 5,
"Minor": 228,
"Patch": 0
"Patch": 1
},
"demands": [
"azureps"
Expand Down
7 changes: 4 additions & 3 deletions Tasks/WindowsMachineFileCopyV1/RoboCopyJob.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ param (
[object]$credential,
[string]$cleanTargetBeforeCopy,
[string]$additionalArguments,
[bool]$useSanitizerActivate = $false
[string]$useSanitizerActivate
)

$sourcePath = $sourcePath.Trim().TrimEnd('\', '/')
Expand Down Expand Up @@ -216,9 +216,10 @@ param (

$robocopyParameters = Get-RoboCopyParameters -additionalArguments $additionalArguments -fileCopy:$isFileCopy -clean:$doCleanUp

if ($useSanitizerActivate) {
if ($useSanitizerActivate -eq "true") {
# Splitting arguments on space, but not on space inside quotes
$sanitizedArguments = [regex]::Split($robocopyParameters, ' (?=(?:[^"]|"[^"]*")*$)')
& robocopy $sourceDirectory $destinationNetworkPath $filesToCopy $sanitizedArguments
& robocopy "$sourceDirectory" "$destinationNetworkPath" "$filesToCopy" $sanitizedArguments
} else {
$command = "robocopy `"$sourceDirectory`" `"$destinationNetworkPath`" `"$filesToCopy`" $robocopyParameters"
Invoke-Expression $command
Expand Down
2 changes: 1 addition & 1 deletion Tasks/WindowsMachineFileCopyV1/task.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 1,
"Minor": 228,
"Patch": 0
"Patch": 1
},
"minimumAgentVersion": "1.104.0",
"groups": [
Expand Down
2 changes: 1 addition & 1 deletion Tasks/WindowsMachineFileCopyV1/task.loc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 1,
"Minor": 228,
"Patch": 0
"Patch": 1
},
"minimumAgentVersion": "1.104.0",
"groups": [
Expand Down
7 changes: 4 additions & 3 deletions Tasks/WindowsMachineFileCopyV2/RoboCopyJob.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ param (
[string]$cleanTargetBeforeCopy,
[string]$additionalArguments,
[string]$scriptRoot,
[bool]$useSanitizerActivate = $false
[string]$useSanitizerActivate
)
Import-Module "$scriptRoot\ps_modules\VstsTaskSdk"
Import-VstsLocStrings -LiteralPath $scriptRoot/Task.json
Expand Down Expand Up @@ -238,9 +238,10 @@ param (
{
$robocopyParameters = Get-RoboCopyParameters -additionalArguments $additionalArguments -fileCopy:$isFileCopy

if ($useSanitizerActivate) {
if ($useSanitizerActivate -eq "true") {
# Splitting arguments on space, but not on space inside quotes
$sanitizedArguments = [regex]::Split($robocopyParameters, ' (?=(?:[^"]|"[^"]*")*$)')
& robocopy $sourceDirectory $destinationNetworkPath $filesToCopy $sanitizedArguments
& robocopy "$sourceDirectory" "$destinationNetworkPath" "$filesToCopy" $sanitizedArguments
} else {
$command = "robocopy `"$sourceDirectory`" `"$destinationNetworkPath`" `"$filesToCopy`" $robocopyParameters"
Invoke-Expression $command
Expand Down
2 changes: 1 addition & 1 deletion Tasks/WindowsMachineFileCopyV2/task.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 2,
"Minor": 228,
"Patch": 0
"Patch": 1
},
"releaseNotes": "What's new in Version 2.0: <br/>&nbsp;&nbsp;Proxy support is being added. <br/>&nbsp;&nbsp; Removed support of legacy DTL machines.",
"minimumAgentVersion": "1.104.0",
Expand Down
2 changes: 1 addition & 1 deletion Tasks/WindowsMachineFileCopyV2/task.loc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 2,
"Minor": 228,
"Patch": 0
"Patch": 1
},
"releaseNotes": "ms-resource:loc.releaseNotes",
"minimumAgentVersion": "1.104.0",
Expand Down
Loading

0 comments on commit 61960ba

Please sign in to comment.