diff --git a/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Publish-NewServiceFabricApplication.ps1 b/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Publish-NewServiceFabricApplication.ps1 index 81bfcff8ae08..2eb7b99533ad 100644 --- a/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Publish-NewServiceFabricApplication.ps1 +++ b/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Publish-NewServiceFabricApplication.ps1 @@ -144,16 +144,7 @@ $ApplicationManifestPath = "$AppPkgPathToUse\ApplicationManifest.xml" - try - { - $global:operationId = $SF_Operations.TestClusterConnection - [void](Test-ServiceFabricClusterConnection) - } - catch - { - Write-Warning (Get-VstsLocString -Key SFSDK_UnableToVerifyClusterConnection) - throw - } + Test-ServiceFabricClusterConnectionAction # If ApplicationName is not specified on command line get application name from Application Parameter file. if (!$ApplicationName) @@ -259,8 +250,7 @@ Write-Host (Get-VstsLocString -Key SFSDK_CopyingAppToImageStore) # Get image store connection string - $global:operationId = $SF_Operations.GetClusterManifest - $clusterManifestText = Get-ServiceFabricClusterManifest + $clusterManifestText = Get-ServiceFabricClusterManifestAction $imageStoreConnectionString = Get-ImageStoreConnectionStringFromClusterManifest ([xml] $clusterManifestText) $applicationPackagePathInImageStore = $names.ApplicationTypeName @@ -310,9 +300,8 @@ Write-Host (Get-VstsLocString -Key SFSDK_RegisterAppType) Register-ServiceFabricApplicationTypeAction -RegisterParameters $registerParameters -ApplicationTypeName $names.ApplicationTypeName -ApplicationTypeVersion $names.ApplicationTypeVersion - $global:operationId = $SF_Operations.RemoveApplicationPackage Write-Host (Get-VstsLocString -Key SFSDK_RemoveAppPackage) - Remove-ServiceFabricApplicationPackage -ApplicationPackagePathInImageStore $applicationPackagePathInImageStore -ImageStoreConnectionString $imageStoreConnectionString + Remove-ServiceFabricApplicationPackageAction -ApplicationPackagePathInImageStore $applicationPackagePathInImageStore -ImageStoreConnectionString $imageStoreConnectionString } } diff --git a/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Publish-UpgradedServiceFabricApplication.ps1 b/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Publish-UpgradedServiceFabricApplication.ps1 index 09a534269ec3..fbb922afc77c 100644 --- a/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Publish-UpgradedServiceFabricApplication.ps1 +++ b/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Publish-UpgradedServiceFabricApplication.ps1 @@ -185,17 +185,7 @@ function Publish-UpgradedServiceFabricApplication return } - - try - { - $global:operationId = $SF_Operations.TestClusterConnection - [void](Test-ServiceFabricClusterConnection) - } - catch - { - Write-Warning (Get-VstsLocString -Key SFSDK_UnableToVerifyClusterConnection) - throw - } + Test-ServiceFabricClusterConnectionAction $ApplicationTypeAlreadyRegistered = $false if ($Action.Equals('RegisterAndUpgrade') -or $Action.Equals('Register')) @@ -235,14 +225,12 @@ function Publish-UpgradedServiceFabricApplication if (!$reg -or !$ApplicationTypeAlreadyRegistered) { # Get image store connection string - $global:operationId = $SF_Operations.GetClusterManifest - $clusterManifestText = Get-ServiceFabricClusterManifest + $clusterManifestText = Get-ServiceFabricClusterManifestAction $imageStoreConnectionString = Get-ImageStoreConnectionStringFromClusterManifest ([xml] $clusterManifestText) if (!$SkipPackageValidation) { - $global:operationId = $SF_Operations.TestApplicationPackage - $packageValidationSuccess = (Test-ServiceFabricApplicationPackage $AppPkgPathToUse -ImageStoreConnectionString $imageStoreConnectionString) + $packageValidationSuccess = (Test-ServiceFabricApplicationPackageAction -AppPkgPath $AppPkgPathToUse -ImageStoreConnectionString $imageStoreConnectionString) if (!$packageValidationSuccess) { $errMsg = (Get-VstsLocString -Key SFSDK_PackageValidationFailed -ArgumentList $ApplicationPackagePath) @@ -331,8 +319,7 @@ function Publish-UpgradedServiceFabricApplication } Write-Host (Get-VstsLocString -Key SFSDK_StartAppUpgrade) - $global:operationId = $SF_Operations.StartApplicationUpgrade - Start-ServiceFabricApplicationUpgrade @UpgradeParameters + Start-ServiceFabricApplicationUpgradeAction -UpgradeParameters $UpgradeParameters } catch { diff --git a/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Utilities.ps1 b/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Utilities.ps1 index 2505a325c1d5..5e589faa4ee6 100644 --- a/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Utilities.ps1 +++ b/Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Utilities.ps1 @@ -240,7 +240,9 @@ function Get-ServiceFabricApplicationUpgradeAction ) $global:operationId = $SF_Operations.GetApplicationUpgradeStatus - return Get-ServiceFabricApplicationUpgrade -ApplicationName $ApplicationName + $getUpgradeAction = { Get-ServiceFabricApplicationUpgrade -ApplicationName $ApplicationName } + return Invoke-ActionWithDefaultRetries -Action $getUpgradeAction ` + -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingGetApplicationUpgrade) } function Wait-ServiceFabricApplicationUpgradeAction @@ -538,8 +540,6 @@ function New-ServiceFabricApplicationAction ) $global:operationId = $SF_Operations.CreateNewApplication - - $createAction = { New-ServiceFabricApplication -ApplicationName $ApplicationName -ApplicationTypeName $ApplicationTypeName -ApplicationTypeVersion $ApplicationTypeVersion -ApplicationParameter $ApplicationParameter } $exceptionRetryEvaluator = { param($ex) @@ -564,11 +564,108 @@ function New-ServiceFabricApplicationAction { Write-Host (Get-VstsLocString -Key SFSDK_CreateApplicationFailed) # print application health status if create did not succeed - Trace-ServiceFabricApplicationHealth + Trace-ServiceFabricApplicationHealth -ApplicationName $ApplicationName + throw + } +} + +function Start-ServiceFabricApplicationUpgradeAction +{ + Param ( + [hashtable] + $UpgradeParameters + ) + + $global:operationId = $SF_Operations.StartApplicationUpgrade + $startAction = { Start-ServiceFabricApplicationUpgrade @UpgradeParameters } + $exceptionRetryEvaluator = { + param($ex) + + # If upgrade already started, don't retry + $upgradeStatus = Get-ServiceFabricApplicationUpgradeAction -ApplicationName $($UpgradeParameters["ApplicationName"]) + if ($upgradeStatus -and ($upgradeStatus.UpgradeState -ne "RollingBackCompleted" -and $upgradeStatus.UpgradeState -ne "RollingForwardCompleted")) + { + return $false + } + + return $true + } + + try + { + Invoke-ActionWithDefaultRetries -Action $startAction ` + -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingUpgradeApplication) ` + -ExceptionRetryEvaluator $exceptionRetryEvaluator ` + -RetryableExceptions @("System.Fabric.FabricTransientException", "System.TimeoutException") + } + catch + { + # print application health status if starting upgrade did not succeed + Trace-ServiceFabricApplicationHealth -ApplicationName $($UpgradeParameters["ApplicationName"]) + throw + } +} + +function Test-ServiceFabricClusterConnectionAction +{ + try + { + $global:operationId = $SF_Operations.TestClusterConnection + $testAction = { [void](Test-ServiceFabricClusterConnection) } + Invoke-ActionWithDefaultRetries -Action $testAction ` + -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingTestClusterConnection) ` + -RetryableExceptions @("System.Fabric.FabricTransientException", "System.TimeoutException") + } + catch + { + Write-Warning (Get-VstsLocString -Key SFSDK_UnableToVerifyClusterConnection) throw } } +function Test-ServiceFabricApplicationPackageAction +{ + Param ( + [string] + $AppPkgPath, + + [string] + $ImageStoreConnectionString + ) + + $global:operationId = $SF_Operations.TestApplicationPackage + $testAction = { Test-ServiceFabricApplicationPackage -ApplicationPackagePath $AppPkgPath -ImageStoreConnectionString $ImageStoreConnectionString } + return Invoke-ActionWithDefaultRetries -Action $testAction ` + -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingTestAppPackage) ` + -RetryableExceptions @("System.Fabric.FabricTransientException", "System.TimeoutException") +} + +function Get-ServiceFabricClusterManifestAction +{ + $global:operationId = $SF_Operations.GetClusterManifest + $manifestAction = { Get-ServiceFabricClusterManifest } + return Invoke-ActionWithDefaultRetries -Action $manifestAction ` + -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingGetClusterManifest) ` + -RetryableExceptions @("System.Fabric.FabricTransientException", "System.TimeoutException") +} + +function Remove-ServiceFabricApplicationPackageAction +{ + Param ( + [string] + $ApplicationPackagePathInImageStore, + + [string] + $ImageStoreConnectionString + ) + + $global:operationId = $SF_Operations.RemoveApplicationPackage + $removeAction = { Remove-ServiceFabricApplicationPackage -ApplicationPackagePathInImageStore $ApplicationPackagePathInImageStore -ImageStoreConnectionString $ImageStoreConnectionString } + Invoke-ActionWithDefaultRetries -Action $removeAction ` + -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingRemoveApplicationPackage) ` + -RetryableExceptions @("System.Fabric.FabricTransientException", "System.TimeoutException") +} + function Trace-ServiceFabricClusterHealth { try diff --git a/Tasks/ServiceFabricDeployV1/Strings/resources.resjson/en-US/resources.resjson b/Tasks/ServiceFabricDeployV1/Strings/resources.resjson/en-US/resources.resjson index d08cb99a8092..a182eb90de2a 100644 --- a/Tasks/ServiceFabricDeployV1/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/ServiceFabricDeployV1/Strings/resources.resjson/en-US/resources.resjson @@ -135,5 +135,11 @@ "loc.messages.SFSDK_RetryingUnregisterApplicationType": "Retrying unregister..", "loc.messages.SFSDK_RetryingRemoveApplication": "Retrying remove application..", "loc.messages.SFSDK_RetryingCreateApplication": "Retrying create application..", - "loc.messages.SFSDK_ApplicationHealth": "Getting application health:" + "loc.messages.SFSDK_ApplicationHealth": "Getting application health:", + "loc.messages.SFSDK_RetryingUpgradeApplication": "Retrying application upgrade..", + "loc.messages.SFSDK_RetryingGetApplicationUpgrade": "Getting application upgrade status...", + "loc.messages.SFSDK_RetryingTestClusterConnection": "Testing connection to cluster..", + "loc.messages.SFSDK_RetryingTestAppPackage": "Testing application package..", + "loc.messages.SFSDK_RetryingGetClusterManifest": "Getting cluster manifest..", + "loc.messages.SFSDK_RetryingRemoveApplicationPackage": "Retrying to remove application package.." } \ No newline at end of file diff --git a/Tasks/ServiceFabricDeployV1/Tests/L0.ts b/Tasks/ServiceFabricDeployV1/Tests/L0.ts index 30c9f9bc00ad..02b0e4dd9c8d 100644 --- a/Tasks/ServiceFabricDeployV1/Tests/L0.ts +++ b/Tasks/ServiceFabricDeployV1/Tests/L0.ts @@ -63,5 +63,29 @@ describe('ServiceFabricDeploy Suite', function () { it('Register application type should retry', (done) => { psr.run(path.join(__dirname, 'RegisterApplicationTypeShouldRetry.ps1'), done); }) + it('Unregister application type should retry till success', (done) => { + psr.run(path.join(__dirname, 'UnregisterApplicationTypeShouldRetryTillSuccess.ps1'), done); + }) + it('Unregister application type should retry', (done) => { + psr.run(path.join(__dirname, 'UnregisterApplicationTypeShouldRetry.ps1'), done); + }) + it('Create application type should retry till success', (done) => { + psr.run(path.join(__dirname, 'CreateApplicationShouldRetryTillSuccess.ps1'), done); + }) + it('Create application type should retry', (done) => { + psr.run(path.join(__dirname, 'CreateApplicationShouldRetry.ps1'), done); + }) + it('Remove application type should retry till success', (done) => { + psr.run(path.join(__dirname, 'RemoveApplicationShouldRetryTillSuccess.ps1'), done); + }) + it('Remove application type should retry', (done) => { + psr.run(path.join(__dirname, 'RemoveApplicationShouldRetry.ps1'), done); + }) + it('Start application upgrade should retry till success', (done) => { + psr.run(path.join(__dirname, 'StartApplicationUpgradeShouldRetryTillSuccess.ps1'), done); + }) + it('Start application upgrade should retry', (done) => { + psr.run(path.join(__dirname, 'StartApplicationUpgradeShouldRetry.ps1'), done); + }) } }); \ No newline at end of file diff --git a/Tasks/ServiceFabricDeployV1/Tests/StartApplicationUpgradeShouldRetry.ps1 b/Tasks/ServiceFabricDeployV1/Tests/StartApplicationUpgradeShouldRetry.ps1 new file mode 100644 index 000000000000..a1e1b3e52c67 --- /dev/null +++ b/Tasks/ServiceFabricDeployV1/Tests/StartApplicationUpgradeShouldRetry.ps1 @@ -0,0 +1,62 @@ +[CmdletBinding()] +param() + +Import-Module ServiceFabric +. $PSScriptRoot\..\..\..\Tests\lib\Initialize-Test.ps1 + +$ApplicationTypeName = "app type name" +$ApplicationTypeVersion = "app type version" +$applicationPackagePathInImageStore = "image path" + +$UpgradeParameters = @{ + 'ApplicationTypeName' = $ApplicationTypeName; + 'ApplicationTypeVersion' = $ApplicationTypeVersion + 'ApplicationParameter' = {} +} + +$upgradeStatus = @{ + 'UpgradeState' = 'RollingForwardCompleted' +} +$global:startUpgradeAttempted = 0 +$global:getRetriesAttempted = 0 +$global:appHealthPrinted = $false + +Register-Mock Start-ServiceFabricApplicationUpgrade { + $global:startUpgradeAttempted++ + throw [System.Fabric.FabricTransientException]::new("Could not ping!") +} -- @UpgradeParameters + +Register-Mock Get-ServiceFabricApplicationUpgrade { + $global:getRetriesAttempted++ + + if ($global:getRetriesAttempted -eq 3) + { + $upgradeStatus.UpgradeState = 'RollingForwardCompleted' + return $upgradeStatus + } + + if ($global:getRetriesAttempted -eq 2) + { + return $null + } + + throw [System.Fabric.FabricTransientException]::new("Could not ping!") +} -- -ApplicationName $ApplicationName + +Register-Mock Get-ServiceFabricApplicationHealth { + $global:appHealthPrinted = $true +} + +Register-Mock Start-Sleep {} +Register-Mock Write-VstsTaskError + +# Act +. $PSScriptRoot\..\..\..\Tasks\ServiceFabricDeployV1\ps_modules\PowershellHelpers\Helpers.ps1 +. $PSScriptRoot\..\..\..\Tasks\ServiceFabricDeployV1\ServiceFabricSDK\Utilities.ps1 + +# Act/Assert +Assert-Throws { + Start-ServiceFabricApplicationUpgradeAction -UpgradeParameters $UpgradeParameters +} +Assert-AreEqual 3 $global:startUpgradeAttempted "Number of start upgrade retries not correct" +Assert-AreEqual $true $global:appHealthPrinted "cluster health not printed in case of error" \ No newline at end of file diff --git a/Tasks/ServiceFabricDeployV1/Tests/StartApplicationUpgradeShouldRetryTillSuccess.ps1 b/Tasks/ServiceFabricDeployV1/Tests/StartApplicationUpgradeShouldRetryTillSuccess.ps1 new file mode 100644 index 000000000000..53218418ff7b --- /dev/null +++ b/Tasks/ServiceFabricDeployV1/Tests/StartApplicationUpgradeShouldRetryTillSuccess.ps1 @@ -0,0 +1,60 @@ +[CmdletBinding()] +param() + +Import-Module ServiceFabric +. $PSScriptRoot\..\..\..\Tests\lib\Initialize-Test.ps1 + +$ApplicationTypeName = "app type name" +$ApplicationTypeVersion = "app type version" +$applicationPackagePathInImageStore = "image path" + +$UpgradeParameters = @{ + 'ApplicationTypeName' = $ApplicationTypeName; + 'ApplicationTypeVersion' = $ApplicationTypeVersion + 'ApplicationParameter' = {} +} + +$upgradeStatus = @{ + 'UpgradeState' = 'RollingForwardCompleted' +} +$global:startUpgradeAttempted = 0 +$global:getRetriesAttempted = 0 +$global:appHealthPrinted = $false + +Register-Mock Start-ServiceFabricApplicationUpgrade { + $global:startUpgradeAttempted++ + throw [System.Fabric.FabricTransientException]::new("Could not ping!") +} -- @UpgradeParameters + +Register-Mock Get-ServiceFabricApplicationUpgrade { + $global:getRetriesAttempted++ + + if ($global:getRetriesAttempted -eq 6) + { + $upgradeStatus.UpgradeState = 'RollingForwardInProgress' + return $upgradeStatus + } + + if ($global:getRetriesAttempted -eq 3) + { + $upgradeStatus.UpgradeState = 'RollingForwardCompleted' + return $upgradeStatus + } + + throw [System.Fabric.FabricTransientException]::new("Could not ping!") +} -- -ApplicationName $ApplicationName + +Register-Mock Get-ServiceFabricApplicationHealth { + $global:appHealthPrinted = $true +} + +Register-Mock Start-Sleep {} +Register-Mock Write-VstsTaskError + +# Act +. $PSScriptRoot\..\..\..\Tasks\ServiceFabricDeployV1\ps_modules\PowershellHelpers\Helpers.ps1 +. $PSScriptRoot\..\..\..\Tasks\ServiceFabricDeployV1\ServiceFabricSDK\Utilities.ps1 + +# Act/Assert +Start-ServiceFabricApplicationUpgradeAction -UpgradeParameters $UpgradeParameters +Assert-AreEqual 2 $global:startUpgradeAttempted "Number of start upgrade retries not correct" \ No newline at end of file diff --git a/Tasks/ServiceFabricDeployV1/task.json b/Tasks/ServiceFabricDeployV1/task.json index bfba9f64dcbf..ee6bb7da4d16 100644 --- a/Tasks/ServiceFabricDeployV1/task.json +++ b/Tasks/ServiceFabricDeployV1/task.json @@ -17,7 +17,7 @@ "version": { "Major": 1, "Minor": 7, - "Patch": 15 + "Patch": 16 }, "demands": [ "Cmd" @@ -476,6 +476,12 @@ "SFSDK_RetryingUnregisterApplicationType": "Retrying unregister..", "SFSDK_RetryingRemoveApplication": "Retrying remove application..", "SFSDK_RetryingCreateApplication": "Retrying create application..", - "SFSDK_ApplicationHealth": "Getting application health:" + "SFSDK_ApplicationHealth": "Getting application health:", + "SFSDK_RetryingUpgradeApplication": "Retrying application upgrade..", + "SFSDK_RetryingGetApplicationUpgrade": "Getting application upgrade status...", + "SFSDK_RetryingTestClusterConnection": "Testing connection to cluster..", + "SFSDK_RetryingTestAppPackage": "Testing application package..", + "SFSDK_RetryingGetClusterManifest": "Getting cluster manifest..", + "SFSDK_RetryingRemoveApplicationPackage": "Retrying to remove application package.." } } \ No newline at end of file diff --git a/Tasks/ServiceFabricDeployV1/task.loc.json b/Tasks/ServiceFabricDeployV1/task.loc.json index 8dbd6c4abf78..fbfe3ad6103e 100644 --- a/Tasks/ServiceFabricDeployV1/task.loc.json +++ b/Tasks/ServiceFabricDeployV1/task.loc.json @@ -17,7 +17,7 @@ "version": { "Major": 1, "Minor": 7, - "Patch": 15 + "Patch": 16 }, "demands": [ "Cmd" @@ -476,6 +476,12 @@ "SFSDK_RetryingUnregisterApplicationType": "ms-resource:loc.messages.SFSDK_RetryingUnregisterApplicationType", "SFSDK_RetryingRemoveApplication": "ms-resource:loc.messages.SFSDK_RetryingRemoveApplication", "SFSDK_RetryingCreateApplication": "ms-resource:loc.messages.SFSDK_RetryingCreateApplication", - "SFSDK_ApplicationHealth": "ms-resource:loc.messages.SFSDK_ApplicationHealth" + "SFSDK_ApplicationHealth": "ms-resource:loc.messages.SFSDK_ApplicationHealth", + "SFSDK_RetryingUpgradeApplication": "ms-resource:loc.messages.SFSDK_RetryingUpgradeApplication", + "SFSDK_RetryingGetApplicationUpgrade": "ms-resource:loc.messages.SFSDK_RetryingGetApplicationUpgrade", + "SFSDK_RetryingTestClusterConnection": "ms-resource:loc.messages.SFSDK_RetryingTestClusterConnection", + "SFSDK_RetryingTestAppPackage": "ms-resource:loc.messages.SFSDK_RetryingTestAppPackage", + "SFSDK_RetryingGetClusterManifest": "ms-resource:loc.messages.SFSDK_RetryingGetClusterManifest", + "SFSDK_RetryingRemoveApplicationPackage": "ms-resource:loc.messages.SFSDK_RetryingRemoveApplicationPackage" } } \ No newline at end of file