[Service fabric] Retry for remaining commands

Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Publish-NewServiceFabricApplication.ps1

@@ -144,16 +144,7 @@
 
     $ApplicationManifestPath = "$AppPkgPathToUse\ApplicationManifest.xml"
 
-    try
-    {
-        $global:operationId = $SF_Operations.TestClusterConnection
-        [void](Test-ServiceFabricClusterConnection)
-    }
-    catch
-    {
-        Write-Warning (Get-VstsLocString -Key SFSDK_UnableToVerifyClusterConnection)
-        throw
-    }
+    Test-ServiceFabricClusterConnectionAction
 
     # If ApplicationName is not specified on command line get application name from Application Parameter file.
     if (!$ApplicationName)
@@ -259,8 +250,7 @@
 
             Write-Host (Get-VstsLocString -Key SFSDK_CopyingAppToImageStore)
             # Get image store connection string
-            $global:operationId = $SF_Operations.GetClusterManifest
-            $clusterManifestText = Get-ServiceFabricClusterManifest
+            $clusterManifestText = Get-ServiceFabricClusterManifestAction
             $imageStoreConnectionString = Get-ImageStoreConnectionStringFromClusterManifest ([xml] $clusterManifestText)
 
             $applicationPackagePathInImageStore = $names.ApplicationTypeName
@@ -310,9 +300,8 @@
             Write-Host (Get-VstsLocString -Key SFSDK_RegisterAppType)
             Register-ServiceFabricApplicationTypeAction -RegisterParameters $registerParameters -ApplicationTypeName $names.ApplicationTypeName -ApplicationTypeVersion $names.ApplicationTypeVersion
 
-            $global:operationId = $SF_Operations.RemoveApplicationPackage
             Write-Host (Get-VstsLocString -Key SFSDK_RemoveAppPackage)
-            Remove-ServiceFabricApplicationPackage -ApplicationPackagePathInImageStore $applicationPackagePathInImageStore -ImageStoreConnectionString $imageStoreConnectionString
+            Remove-ServiceFabricApplicationPackageAction -ApplicationPackagePathInImageStore $applicationPackagePathInImageStore -ImageStoreConnectionString $imageStoreConnectionString
         }
     }
 

Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Publish-UpgradedServiceFabricApplication.ps1

@@ -185,17 +185,7 @@ function Publish-UpgradedServiceFabricApplication
         return
     }
 
-
-    try
-    {
-        $global:operationId = $SF_Operations.TestClusterConnection
-        [void](Test-ServiceFabricClusterConnection)
-    }
-    catch
-    {
-        Write-Warning (Get-VstsLocString -Key SFSDK_UnableToVerifyClusterConnection)
-        throw
-    }
+    Test-ServiceFabricClusterConnectionAction
 
     $ApplicationTypeAlreadyRegistered = $false
     if ($Action.Equals('RegisterAndUpgrade') -or $Action.Equals('Register'))
@@ -235,14 +225,12 @@ function Publish-UpgradedServiceFabricApplication
         if (!$reg -or !$ApplicationTypeAlreadyRegistered)
         {
             # Get image store connection string
-            $global:operationId = $SF_Operations.GetClusterManifest
-            $clusterManifestText = Get-ServiceFabricClusterManifest
+            $clusterManifestText = Get-ServiceFabricClusterManifestAction
             $imageStoreConnectionString = Get-ImageStoreConnectionStringFromClusterManifest ([xml] $clusterManifestText)
 
             if (!$SkipPackageValidation)
             {
-                $global:operationId = $SF_Operations.TestApplicationPackage
-                $packageValidationSuccess = (Test-ServiceFabricApplicationPackage $AppPkgPathToUse -ImageStoreConnectionString $imageStoreConnectionString)
+                $packageValidationSuccess = (Test-ServiceFabricApplicationPackageAction -AppPkgPath $AppPkgPathToUse -ImageStoreConnectionString $imageStoreConnectionString)
                 if (!$packageValidationSuccess)
                 {
                     $errMsg = (Get-VstsLocString -Key SFSDK_PackageValidationFailed -ArgumentList $ApplicationPackagePath)
@@ -331,8 +319,7 @@ function Publish-UpgradedServiceFabricApplication
             }
 
             Write-Host (Get-VstsLocString -Key SFSDK_StartAppUpgrade)
-            $global:operationId = $SF_Operations.StartApplicationUpgrade
-            Start-ServiceFabricApplicationUpgrade @UpgradeParameters
+            Start-ServiceFabricApplicationUpgradeAction -UpgradeParameters $UpgradeParameters
         }
         catch
         {

Tasks/ServiceFabricDeployV1/ServiceFabricSDK/Utilities.ps1

@@ -240,7 +240,9 @@ function Get-ServiceFabricApplicationUpgradeAction
     )
 
     $global:operationId = $SF_Operations.GetApplicationUpgradeStatus
-    return Get-ServiceFabricApplicationUpgrade -ApplicationName $ApplicationName
+    $getUpgradeAction = { Get-ServiceFabricApplicationUpgrade -ApplicationName $ApplicationName }
+    return Invoke-ActionWithDefaultRetries -Action $getUpgradeAction `
+        -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingGetApplicationUpgrade)
 }
 
 function Wait-ServiceFabricApplicationUpgradeAction
@@ -538,8 +540,6 @@ function New-ServiceFabricApplicationAction
     )
 
     $global:operationId = $SF_Operations.CreateNewApplication
-
-
     $createAction = { New-ServiceFabricApplication -ApplicationName $ApplicationName -ApplicationTypeName $ApplicationTypeName -ApplicationTypeVersion $ApplicationTypeVersion -ApplicationParameter $ApplicationParameter }
     $exceptionRetryEvaluator = {
         param($ex)
@@ -564,11 +564,108 @@ function New-ServiceFabricApplicationAction
     {
         Write-Host (Get-VstsLocString -Key SFSDK_CreateApplicationFailed)
         # print application health status if create did not succeed
-        Trace-ServiceFabricApplicationHealth
+        Trace-ServiceFabricApplicationHealth -ApplicationName $ApplicationName
+        throw
+    }
+}
+
+function Start-ServiceFabricApplicationUpgradeAction
+{
+    Param (
+        [hashtable]
+        $UpgradeParameters
+    )
+
+    $global:operationId = $SF_Operations.StartApplicationUpgrade
+    $startAction = { Start-ServiceFabricApplicationUpgrade @UpgradeParameters }
+    $exceptionRetryEvaluator = {
+        param($ex)
+
+        # If upgrade already started, don't retry
+        $upgradeStatus = Get-ServiceFabricApplicationUpgradeAction -ApplicationName $($UpgradeParameters["ApplicationName"])
+        if ($upgradeStatus -and ($upgradeStatus.UpgradeState -ne "RollingBackCompleted" -and $upgradeStatus.UpgradeState -ne "RollingForwardCompleted"))
+        {
+            return $false
+        }
+
+        return $true
+    }
+
+    try
+    {
+        Invoke-ActionWithDefaultRetries -Action $startAction `
+            -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingUpgradeApplication) `
+            -ExceptionRetryEvaluator $exceptionRetryEvaluator `
+            -RetryableExceptions @("System.Fabric.FabricTransientException", "System.TimeoutException")
+    }
+    catch
+    {
+        # print application health status if starting upgrade did not succeed
+        Trace-ServiceFabricApplicationHealth -ApplicationName $($UpgradeParameters["ApplicationName"])
+        throw
+    }
+}
+
+function Test-ServiceFabricClusterConnectionAction
+{
+    try
+    {
+        $global:operationId = $SF_Operations.TestClusterConnection
+        $testAction = { [void](Test-ServiceFabricClusterConnection) }
+        Invoke-ActionWithDefaultRetries -Action $testAction `
+            -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingTestClusterConnection) `
+            -RetryableExceptions @("System.Fabric.FabricTransientException", "System.TimeoutException")
+    }
+    catch
+    {
+        Write-Warning (Get-VstsLocString -Key SFSDK_UnableToVerifyClusterConnection)
         throw
     }
 }
 
+function Test-ServiceFabricApplicationPackageAction
+{
+    Param (
+        [string]
+        $AppPkgPath,
+
+        [string]
+        $ImageStoreConnectionString
+    )
+
+    $global:operationId = $SF_Operations.TestApplicationPackage
+    $testAction = { Test-ServiceFabricApplicationPackage -ApplicationPackagePath $AppPkgPath -ImageStoreConnectionString $ImageStoreConnectionString }
+    return Invoke-ActionWithDefaultRetries -Action $testAction `
+        -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingTestAppPackage) `
+        -RetryableExceptions @("System.Fabric.FabricTransientException", "System.TimeoutException")
+}
+
+function Get-ServiceFabricClusterManifestAction
+{
+    $global:operationId = $SF_Operations.GetClusterManifest
+    $manifestAction = { Get-ServiceFabricClusterManifest }
+    return Invoke-ActionWithDefaultRetries -Action $manifestAction `
+        -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingGetClusterManifest) `
+        -RetryableExceptions @("System.Fabric.FabricTransientException", "System.TimeoutException")
+}
+
+function Remove-ServiceFabricApplicationPackageAction
+{
+    Param (
+        [string]
+        $ApplicationPackagePathInImageStore,
+
+        [string]
+        $ImageStoreConnectionString
+    )
+
+    $global:operationId = $SF_Operations.RemoveApplicationPackage
+    $removeAction = { Remove-ServiceFabricApplicationPackage -ApplicationPackagePathInImageStore $ApplicationPackagePathInImageStore -ImageStoreConnectionString $ImageStoreConnectionString }
+    Invoke-ActionWithDefaultRetries -Action $removeAction `
+        -RetryMessage (Get-VstsLocString -Key SFSDK_RetryingRemoveApplicationPackage) `
+        -RetryableExceptions @("System.Fabric.FabricTransientException", "System.TimeoutException")
+}
+
 function Trace-ServiceFabricClusterHealth
 {
     try

Tasks/ServiceFabricDeployV1/Strings/resources.resjson/en-US/resources.resjson

@@ -135,5 +135,11 @@
   "loc.messages.SFSDK_RetryingUnregisterApplicationType": "Retrying unregister..",
   "loc.messages.SFSDK_RetryingRemoveApplication": "Retrying remove application..",
   "loc.messages.SFSDK_RetryingCreateApplication": "Retrying create application..",
-  "loc.messages.SFSDK_ApplicationHealth": "Getting application health:"
+  "loc.messages.SFSDK_ApplicationHealth": "Getting application health:",
+  "loc.messages.SFSDK_RetryingUpgradeApplication": "Retrying application upgrade..",
+  "loc.messages.SFSDK_RetryingGetApplicationUpgrade": "Getting application upgrade status...",
+  "loc.messages.SFSDK_RetryingTestClusterConnection": "Testing connection to cluster..",
+  "loc.messages.SFSDK_RetryingTestAppPackage": "Testing application package..",
+  "loc.messages.SFSDK_RetryingGetClusterManifest": "Getting cluster manifest..",
+  "loc.messages.SFSDK_RetryingRemoveApplicationPackage": "Retrying to remove application package.."
 }

Tasks/ServiceFabricDeployV1/Tests/L0.ts

@@ -63,5 +63,29 @@ describe('ServiceFabricDeploy Suite', function () {
         it('Register application type should retry', (done) => {
             psr.run(path.join(__dirname, 'RegisterApplicationTypeShouldRetry.ps1'), done);
         })
+        it('Unregister application type should retry till success', (done) => {
+            psr.run(path.join(__dirname, 'UnregisterApplicationTypeShouldRetryTillSuccess.ps1'), done);
+        })
+        it('Unregister application type should retry', (done) => {
+            psr.run(path.join(__dirname, 'UnregisterApplicationTypeShouldRetry.ps1'), done);
+        })
+        it('Create application type should retry till success', (done) => {
+            psr.run(path.join(__dirname, 'CreateApplicationShouldRetryTillSuccess.ps1'), done);
+        })
+        it('Create application type should retry', (done) => {
+            psr.run(path.join(__dirname, 'CreateApplicationShouldRetry.ps1'), done);
+        })
+        it('Remove application type should retry till success', (done) => {
+            psr.run(path.join(__dirname, 'RemoveApplicationShouldRetryTillSuccess.ps1'), done);
+        })
+        it('Remove application type should retry', (done) => {
+            psr.run(path.join(__dirname, 'RemoveApplicationShouldRetry.ps1'), done);
+        })
+        it('Start application upgrade should retry till success', (done) => {
+            psr.run(path.join(__dirname, 'StartApplicationUpgradeShouldRetryTillSuccess.ps1'), done);
+        })
+        it('Start application upgrade should retry', (done) => {
+            psr.run(path.join(__dirname, 'StartApplicationUpgradeShouldRetry.ps1'), done);
+        })
     }
 });

Tasks/ServiceFabricDeployV1/Tests/StartApplicationUpgradeShouldRetry.ps1

@@ -0,0 +1,62 @@
+[CmdletBinding()]
+param()
+
+Import-Module ServiceFabric
+. $PSScriptRoot\..\..\..\Tests\lib\Initialize-Test.ps1
+
+$ApplicationTypeName = "app type name"
+$ApplicationTypeVersion = "app type version"
+$applicationPackagePathInImageStore = "image path"
+
+$UpgradeParameters = @{
+    'ApplicationTypeName'    = $ApplicationTypeName;
+    'ApplicationTypeVersion' = $ApplicationTypeVersion
+    'ApplicationParameter'   = {}
+}
+
+$upgradeStatus = @{
+    'UpgradeState' = 'RollingForwardCompleted'
+}
+$global:startUpgradeAttempted = 0
+$global:getRetriesAttempted = 0
+$global:appHealthPrinted = $false
+
+Register-Mock Start-ServiceFabricApplicationUpgrade {
+    $global:startUpgradeAttempted++
+    throw [System.Fabric.FabricTransientException]::new("Could not ping!")
+} -- @UpgradeParameters
+
+Register-Mock Get-ServiceFabricApplicationUpgrade {
+    $global:getRetriesAttempted++
+
+    if ($global:getRetriesAttempted -eq 3)
+    {
+        $upgradeStatus.UpgradeState = 'RollingForwardCompleted'
+        return $upgradeStatus
+    }
+
+    if ($global:getRetriesAttempted -eq 2)
+    {
+        return $null
+    }
+
+    throw [System.Fabric.FabricTransientException]::new("Could not ping!")
+} -- -ApplicationName $ApplicationName
+
+Register-Mock Get-ServiceFabricApplicationHealth {
+    $global:appHealthPrinted = $true
+}
+
+Register-Mock Start-Sleep {}
+Register-Mock Write-VstsTaskError
+
+# Act
+. $PSScriptRoot\..\..\..\Tasks\ServiceFabricDeployV1\ps_modules\PowershellHelpers\Helpers.ps1
+. $PSScriptRoot\..\..\..\Tasks\ServiceFabricDeployV1\ServiceFabricSDK\Utilities.ps1
+
+# Act/Assert
+Assert-Throws {
+    Start-ServiceFabricApplicationUpgradeAction -UpgradeParameters $UpgradeParameters
+}
+Assert-AreEqual 3 $global:startUpgradeAttempted "Number of start upgrade retries not correct"
+Assert-AreEqual $true $global:appHealthPrinted "cluster health not printed in case of error"

Tasks/ServiceFabricDeployV1/Tests/StartApplicationUpgradeShouldRetryTillSuccess.ps1

@@ -0,0 +1,60 @@
+[CmdletBinding()]
+param()
+
+Import-Module ServiceFabric
+. $PSScriptRoot\..\..\..\Tests\lib\Initialize-Test.ps1
+
+$ApplicationTypeName = "app type name"
+$ApplicationTypeVersion = "app type version"
+$applicationPackagePathInImageStore = "image path"
+
+$UpgradeParameters = @{
+    'ApplicationTypeName'    = $ApplicationTypeName;
+    'ApplicationTypeVersion' = $ApplicationTypeVersion
+    'ApplicationParameter'   = {}
+}
+
+$upgradeStatus = @{
+    'UpgradeState' = 'RollingForwardCompleted'
+}
+$global:startUpgradeAttempted = 0
+$global:getRetriesAttempted = 0
+$global:appHealthPrinted = $false
+
+Register-Mock Start-ServiceFabricApplicationUpgrade {
+    $global:startUpgradeAttempted++
+    throw [System.Fabric.FabricTransientException]::new("Could not ping!")
+} -- @UpgradeParameters
+
+Register-Mock Get-ServiceFabricApplicationUpgrade {
+    $global:getRetriesAttempted++
+
+    if ($global:getRetriesAttempted -eq 6)
+    {
+        $upgradeStatus.UpgradeState = 'RollingForwardInProgress'
+        return $upgradeStatus
+    }
+
+    if ($global:getRetriesAttempted -eq 3)
+    {
+        $upgradeStatus.UpgradeState = 'RollingForwardCompleted'
+        return $upgradeStatus
+    }
+
+    throw [System.Fabric.FabricTransientException]::new("Could not ping!")
+} -- -ApplicationName $ApplicationName
+
+Register-Mock Get-ServiceFabricApplicationHealth {
+    $global:appHealthPrinted = $true
+}
+
+Register-Mock Start-Sleep {}
+Register-Mock Write-VstsTaskError
+
+# Act
+. $PSScriptRoot\..\..\..\Tasks\ServiceFabricDeployV1\ps_modules\PowershellHelpers\Helpers.ps1
+. $PSScriptRoot\..\..\..\Tasks\ServiceFabricDeployV1\ServiceFabricSDK\Utilities.ps1
+
+# Act/Assert
+Start-ServiceFabricApplicationUpgradeAction -UpgradeParameters $UpgradeParameters
+Assert-AreEqual 2 $global:startUpgradeAttempted "Number of start upgrade retries not correct"