Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fixed single file blob upload #7222

Merged
merged 10 commits into from
May 18, 2018
40 changes: 30 additions & 10 deletions Tasks/AzureFileCopyV2/AzureFileCopyRemoteJob.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,22 @@ $AzureFileCopyRemoteJob = {
}
}

function Write-LogsAndCleanup
{
[CmdletBinding()]
param(
[Nullable[bool]]$isLogsPresent,
[AllowEmptyString()][string]$logFilePath,
[AllowEmptyString()][string]$azCopyLocation
)

# Print AzCopy.exe verbose logs
Get-AzCopyVerboseLogs -isLogsPresent $isLogsPresent -logFilePath $logFilePath -ErrorAction SilentlyContinue

# Delete AzCopy tool folder
Remove-AzCopyFolder -azCopyLocation $azCopyLocation -ErrorAction SilentlyContinue
}

function Get-AzCopyVerboseLogs
{
[CmdletBinding()]
Expand Down Expand Up @@ -51,6 +67,8 @@ $AzureFileCopyRemoteJob = {

try
{
$useDefaultArguments = ($additionalArguments -eq "")

$azCopyToolFileNames = $azCopyToolFileNamesString.Split(";")
$azCopyToolFileContents = $azCopyToolFileContentsString.Split(";")

Expand All @@ -72,15 +90,21 @@ $AzureFileCopyRemoteJob = {

if($CleanTargetBeforeCopy)
{
Get-ChildItem -Path $targetPath -Recurse -Force | Remove-Item -Force -Recurse
Write-DetailLogs "Destination location cleaned"
if (Test-Path $targetPath -PathType Container)
{
Get-ChildItem -Path $targetPath -Recurse -Force | Remove-Item -Force -Recurse
Write-DetailLogs "Destination location cleaned"
}
else
{
Write-DetailLogs "Folder at path $targtPath not found for cleanup."
}
}

$azCopyExeLocation = Join-Path -Path $azCopyDestinationPath -ChildPath "AzCopy.exe"

$logFileName = "AzCopyVerbose_" + [guid]::NewGuid() + ".log"
$logFilePath = Join-Path -Path $azCopyDestinationPath -ChildPath $logFileName
$useDefaultArguments = ($additionalArguments -eq "")

if($useDefaultArguments)
{
Expand All @@ -94,9 +118,9 @@ $AzureFileCopyRemoteJob = {
$additionalArguments = "/Z:`"$azCopyDestinationPath`" /V:`"$logFilePath`" /S /Y"
}

Write-DetailLogs "Executing command: & `"$azCopyExeLocation`" /Source:$containerURL /Dest:`"$targetPath`" /SourceSAS:`"*****`" $additionalArguments"
Write-DetailLogs "Executing command: & `"$azCopyExeLocation`" /Source:`"$containerURL`" /Dest:`"$targetPath`" /SourceSAS:`"*****`" $additionalArguments"

$azCopyCommand = "& `"$azCopyExeLocation`" /Source:$containerURL /Dest:`"$targetPath`" /SourceSAS:`"$containerSasToken`" $additionalArguments"
$azCopyCommand = "& `"$azCopyExeLocation`" /Source:`"$containerURL`" /Dest:`"$targetPath`" /SourceSAS:`"$containerSasToken`" $additionalArguments"
Invoke-Expression $azCopyCommand
}
catch
Expand All @@ -106,10 +130,6 @@ $AzureFileCopyRemoteJob = {
}
finally
{
# Print AzCopy.exe verbose logs
Get-AzCopyVerboseLogs -isLogsPresent $useDefaultArguments -logFilePath $logFilePath -ErrorAction SilentlyContinue

# Delete AzCopy tool folder
Remove-AzCopyFolder -azCopyLocation $azCopyDestinationPath -ErrorAction SilentlyContinue
Write-LogsAndCleanup -isLogsPresent $useDefaultArguments -logFilePath "$logFilePath" -azCopyLocation "$azCopyDestinationPath" -ErrorAction SilentlyContinue
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
"loc.input.label.TargetPath": "Destination Folder",
"loc.input.help.TargetPath": "Local path on the target machines for copying the files from the source. Environment variable can be used like $env:windir\\BudgetIT\\Web.",
"loc.input.label.AdditionalArgumentsForBlobCopy": "Optional Arguments (for uploading files to blob)",
"loc.input.help.AdditionalArgumentsForBlobCopy": "Optional AzCopy.exe arguments that will be applied when uploading to blob like, /NC:10. If no optional arguments are specified here, the following optional arguments will be added by default.<br> /Y, /SetContentType, /Z, /V,<br> /S (only if container name is not $root),<br> /BlobType:page (only if specified storage account is a premium account)",
"loc.input.help.AdditionalArgumentsForBlobCopy": "Optional AzCopy.exe arguments that will be applied when uploading to blob like, /NC:10. If no optional arguments are specified here, the following optional arguments will be added by default.<br> /Y, /SetContentType, /Z, /V,<br> /S (only if container name is not $root),<br> /BlobType:page (only if specified storage account is a premium account).<br> If source path is a file, /Pattern will always be added irrespective of whether or not you have specified optional arguments.",
"loc.input.label.AdditionalArgumentsForVMCopy": "Optional Arguments (for downloading files to VM)",
"loc.input.help.AdditionalArgumentsForVMCopy": "Optional AzCopy.exe arguments that will be applied when downloading to VM like, /NC:10. If no optional arguments are specified here, the following optional arguments will be added by default.<br> /Y, /S, /Z, /V",
"loc.input.label.enableCopyPrerequisites": "Enable Copy Prerequisites",
Expand Down Expand Up @@ -93,5 +93,6 @@
"loc.messages.AFC_BlobStorageNotFound": "Storage account: {0} not found. Please specify existing storage account",
"loc.messages.AFC_RootContainerAndDirectory": "'/S' option is not valid for $root containers.",
"loc.messages.AFC_RedirectResponseInvalidStatusCode": "The HTTP response code: '{0}' is not a valid redirect status code",
"loc.messages.AFC_RedirectResponseLocationHeaderIsNull": "Redirect response location header is null."
"loc.messages.AFC_RedirectResponseLocationHeaderIsNull": "Redirect response location header is null.",
"loc.messages.AFC_AzCopyBlobUploadNonZeroExitCode": "AzCopy.exe exited with non-zero exit code while uploading files to blob storage."
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ $invalidInputStorageAccount = "invalidInputStorageAccount"
$exceptionMessage = "Exception thrown"

Register-Mock Write-Telemetry { }
Register-Mock Test-Path { return $false }

# Test 1 "Should throw if Invoke-Expression fails"
Register-Mock Invoke-Expression { throw $exceptionMessage }
Expand All @@ -31,6 +32,7 @@ Assert-WasCalled Remove-AzureContainer -Times 1
# Test 3 "Success in Upload blob destination"
Unregister-Mock Invoke-Expression
Register-Mock Invoke-Expression { return $succeededCopyResponse }
$LASTEXITCODE = 0

Upload-FilesToAzureContainer -sourcePath $validInputSourcePath -storageAccountName $validInputStorageAccount -containerName $validInputContainerName `
-blobPrefix $validInputBlobPrefix -storageKey $validStorageKey -azCopyLocation $validAzCopyLocation -destinationType $validInputAzureBlobDestinationType
Expand Down
30 changes: 24 additions & 6 deletions Tasks/AzureFileCopyV2/Utility.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -247,16 +247,34 @@ function Upload-FilesToAzureContainer
{
Write-Output (Get-VstsLocString -Key "AFC_UploadFilesStorageAccount" -ArgumentList $sourcePath, $storageAccountName, $containerName, $blobPrefix)

$resolvedSourcePath = $sourcePath

# Check if source path is a file. If yes, then add /Pattern additional argument.
if(Test-Path -Path $sourcePath -PathType Leaf)
{
$fileInfo = Get-Item $sourcePath
$resolvedSourcePath = $fileInfo.Directory.FullName
$additionalArguments += " /Pattern:`"$($fileInfo.Name)`""
}

$blobPrefix = $blobPrefix.Trim()
$containerURL = [string]::Format("{0}/{1}/{2}", $blobStorageEndpoint.Trim("/"), $containerName, $blobPrefix).Trim("/")
$azCopyExeLocation = Join-Path -Path $azCopyLocation -ChildPath "AzCopy.exe"

Write-Verbose "Executing command: & `"$azCopyExeLocation`" /Source:$sourcePath /Dest:$containerURL /DestKey:`"*****`" $additionalArguments"
Write-Verbose "Executing command: & `"$azCopyExeLocation`" /Source:`"$resolvedSourcePath`" /Dest:`"$containerURL`" /DestKey:`"*****`" $additionalArguments"
Copy link
Member

@arjgupta arjgupta May 17, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it will be better if you write this with ##[command] and write it to output and not the verbose stream. Will be better experience. And no need to repeat the command here and in the next line. create it once, and then use it wherever. Later if this command changes, you will need to update at only place.


$uploadToBlobCommand = "& `"$azCopyExeLocation`" /Source:$sourcePath /Dest:$containerURL /DestKey:`"$storageKey`" $additionalArguments"
$uploadToBlobCommand = "& `"$azCopyExeLocation`" /Source:`"$resolvedSourcePath`" /Dest:`"$containerURL`" /DestKey:`"$storageKey`" $additionalArguments"

Invoke-Expression $uploadToBlobCommand
Write-Output (Get-VstsLocString -Key "AFC_UploadFileSuccessful" -ArgumentList $sourcePath, $storageAccountName, $containerName, $blobPrefix)

if($LASTEXITCODE -eq 0)
{
Write-Output (Get-VstsLocString -Key "AFC_UploadFileSuccessful" -ArgumentList $sourcePath, $storageAccountName, $containerName, $blobPrefix)
}
else
{
throw (Get-VstsLocString -Key "AFC_AzCopyBlobUploadNonZeroExitCode")
}
}
catch
{
Expand All @@ -275,16 +293,16 @@ function Upload-FilesToAzureContainer
}
finally
{
Handle-AzCopyLogs -isLogsPresent $useDefaultArguments -logsFilePath $azCopyLogFilePath -ErrorAction SilentlyContinue
Handle-AzCopyLogs -isLogsPresent $useDefaultArguments -logsFilePath "$azCopyLogFilePath" -ErrorAction SilentlyContinue
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit, you do not need to enclose it in double quotes.

}
}

function Handle-AzCopyLogs
{
[CmdletBinding()]
param(
[bool]$isLogsPresent,
[string]$logsFilePath
[Nullable[bool]]$isLogsPresent,
[AllowEmptyString()][string]$logsFilePath
)

if($isLogsPresent)
Expand Down
5 changes: 3 additions & 2 deletions Tasks/AzureFileCopyV2/task.json
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@
"label": "Optional Arguments (for uploading files to blob)",
"required": false,
"defaultValue": "",
"helpMarkDown": "Optional AzCopy.exe arguments that will be applied when uploading to blob like, /NC:10. If no optional arguments are specified here, the following optional arguments will be added by default.<br> /Y, /SetContentType, /Z, /V,<br> /S (only if container name is not $root),<br> /BlobType:page (only if specified storage account is a premium account)"
"helpMarkDown": "Optional AzCopy.exe arguments that will be applied when uploading to blob like, /NC:10. If no optional arguments are specified here, the following optional arguments will be added by default.<br> /Y, /SetContentType, /Z, /V,<br> /S (only if container name is not $root),<br> /BlobType:page (only if specified storage account is a premium account).<br> If source path is a file, /Pattern will always be added irrespective of whether or not you have specified optional arguments."
},
{
"name": "AdditionalArgumentsForVMCopy",
Expand Down Expand Up @@ -344,6 +344,7 @@
"AFC_BlobStorageNotFound": "Storage account: {0} not found. Please specify existing storage account",
"AFC_RootContainerAndDirectory": "'/S' option is not valid for $root containers.",
"AFC_RedirectResponseInvalidStatusCode": "The HTTP response code: '{0}' is not a valid redirect status code",
"AFC_RedirectResponseLocationHeaderIsNull": "Redirect response location header is null."
"AFC_RedirectResponseLocationHeaderIsNull": "Redirect response location header is null.",
"AFC_AzCopyBlobUploadNonZeroExitCode": "AzCopy.exe exited with non-zero exit code while uploading files to blob storage."
}
}
3 changes: 2 additions & 1 deletion Tasks/AzureFileCopyV2/task.loc.json
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@
"AFC_BlobStorageNotFound": "ms-resource:loc.messages.AFC_BlobStorageNotFound",
"AFC_RootContainerAndDirectory": "ms-resource:loc.messages.AFC_RootContainerAndDirectory",
"AFC_RedirectResponseInvalidStatusCode": "ms-resource:loc.messages.AFC_RedirectResponseInvalidStatusCode",
"AFC_RedirectResponseLocationHeaderIsNull": "ms-resource:loc.messages.AFC_RedirectResponseLocationHeaderIsNull"
"AFC_RedirectResponseLocationHeaderIsNull": "ms-resource:loc.messages.AFC_RedirectResponseLocationHeaderIsNull",
"AFC_AzCopyBlobUploadNonZeroExitCode": "ms-resource:loc.messages.AFC_AzCopyBlobUploadNonZeroExitCode"
}
}