upgrade with diff package (#5962)

* upgrade with diff package

* resolve comments

* remove Compare-Version logic since adding build number to version is on purpose

* add unit test and resolve comments

* resolve comments 2

* avoid bumping major version by setting diff package path to env:Temp

Tasks/ServiceFabricDeploy/Create-DiffPackage.psm1

@@ -0,0 +1,151 @@
+function Create-DiffPackage
+{
+    [CmdletBinding()]
+    param(
+        [parameter(Mandatory=$true)][String] $ApplicationName,
+        [parameter(Mandatory=$true)][String] $ApplicationPackagePath,
+        [parameter(Mandatory=$true)] $ConnectedServiceEndpoint,
+        [parameter(Mandatory=$true)][Hashtable] $ClusterConnectionParameters
+    )
+
+    Trace-VstsEnteringInvocation $MyInvocation
+    try {
+        Write-Host (Get-VstsLocString -Key DIFFPKG_CreatingDiffPackage)
+
+        Import-Module $PSScriptRoot\ps_modules\ServiceFabricHelpers
+        Import-Module $PSScriptRoot\ps_modules\PowershellHelpers
+
+        $appManifestName = "ApplicationManifest.xml"
+        $localAppManifestPath = Join-Path $ApplicationPackagePath $appManifestName
+        $localAppManifestXml = [XML](Get-Content $localAppManifestPath)
+        $applicationTypeName = $localAppManifestXml.ApplicationManifest.ApplicationTypeName
+        $localAppTypeVersion = $localAppManifestXml.ApplicationManifest.ApplicationTypeVersion
+
+        $app = Get-ServiceFabricApplication -ApplicationName $ApplicationName
+
+        # If $app is null, it means the application does not exist in the cluster. Diff Package is equal to Full Package. Use Full Package to do deployment
+        if (!$app -or $app.ApplicationTypeName -ne $applicationTypeName)
+        {
+            Write-Host (Get-VstsLocString -Key DIFFPKG_ApplicationDoesNotExist -ArgumentList @($ApplicationName, $ConnectedServiceEndpoint.Url))
+            Return
+        }
+
+        $clusterAppTypeVersion = $app.ApplicationTypeVersion
+
+        # If the ApplicationTypeVersion of the Application is not upgraded, no diff package is made because there is no need
+        if ($clusterAppTypeVersion -eq $localAppTypeVersion)
+        {
+            Write-Host (Get-VstsLocString -Key DIFFPKG_ApplicationIsNotChanged -ArgumentList @($ApplicationName, $clusterAppTypeVersion, $ConnectedServiceEndpoint.Url))
+            Return
+        }
+
+        $diffPackagePath = Join-Path $env:Temp "DiffPackage"
+        if (Test-Path -PathType Container -Path $diffPackagePath)
+        {
+            Remove-Item -Path $diffPackagePath -Recurse -Force
+        }
+        $diffPackagePath = New-Item -ItemType Directory -Path $diffPackagePath -Force
+        $diffPkgAppManifestPath = Join-Path $diffPackagePath $appManifestName
+
+        # Get the service types from the cluster
+        $serviceTypes = Get-ServiceFabricServiceType -ApplicationTypeName $applicationTypeName -ApplicationTypeVersion $clusterAppTypeVersion
+        # Pack the service manifest names into an array
+        $clusterServiceManifestNames = $serviceTypes.ServiceManifestName
+
+        # If $clusterServiceManifestNames is null, it means no service types are registered. Diff Package is equal to Full Package. Use Full Package to do deployment
+        if (!$clusterServiceManifestNames)
+        {
+            Write-Host (Get-VstsLocString -Key DIFFPKG_NoServicesRunning -ArgumentList @($ApplicationName, $ConnectedServiceEndpoint.Url))
+            Return
+        }
+
+        Write-Host (Get-VstsLocString -Key DIFFPKG_CopyingToDiffPackge -ArgumentList @($localAppManifestPath, $diffPkgAppManifestPath))
+        Copy-Item $localAppManifestPath $diffPkgAppManifestPath -Force
+
+        # Get the service manifests from the cluster
+        $clusterServiceManifestByName = @{}
+        foreach ($clusterServiceManifestName in $clusterServiceManifestNames)
+        {
+            $clusterServiceManifestContent = Get-ServiceFabricServiceManifest -ApplicationTypeName $applicationTypeName -ApplicationTypeVersion $clusterAppTypeVersion -ServiceManifestName $clusterServiceManifestName
+            $clusterServiceManifestByName[$clusterServiceManifestName] = [XML]$clusterServiceManifestContent
+        }
+
+        foreach ($serviceManifestImport in $localAppManifestXml.ApplicationManifest.ServiceManifestImport)
+        {
+            # Open the service manifest associated with the current ServiceManifestImport element of the local ApplicationManifest
+            $serviceManifestName = "ServiceManifest.xml"
+            $localServiceManifestName = $serviceManifestImport.ServiceManifestRef.ServiceManifestName
+            $localServiceManifestVersion = $serviceManifestImport.ServiceManifestRef.ServiceManifestVersion
+            $localServicePkgPath = Join-Path $ApplicationPackagePath $localServiceManifestName
+            $localServiceManifestPath = [System.IO.Path]::Combine($localServicePkgPath, $serviceManifestName)
+            $localServiceManifest = ([XML](Get-Content $localServiceManifestPath)).ServiceManifest
+            $diffServicePkgPath = [System.IO.Path]::Combine($diffPackagePath, $localServiceManifestName)
+            $clusterServiceManifest = $clusterServiceManifestByName[$localServiceManifestName].ServiceManifest
+
+            # If there's no matching manifest from the cluster it means this is a newly added service that doesn't exist yet on the cluster. 
+            if (!$clusterServiceManifest)
+            {
+                # Copy this service and all the children
+                Write-Host (Get-VstsLocString -Key DIFFPKG_ServiceDoesNotExist -ArgumentList @($localServiceManifestName, $ApplicationName, $ConnectedServiceEndpoint.Url))
+                Copy-Item $localServicePkgPath $diffServicePkgPath -Recurse
+                continue
+            }
+
+            # If the Version of the Service is not changed, don't include the service in the diff package
+            if ($clusterServiceManifest.Version -eq $localServiceManifestVersion)
+            {
+                Write-Host (Get-VstsLocString -Key DIFFPKG_ServiceIsNotChanged -ArgumentList @($localServiceManifestName, $ApplicationName, $clusterServiceManifest.Version, $ConnectedServiceEndpoint.Url))
+                continue
+            }
+            Write-Host (Get-VstsLocString -Key DIFFPKG_CreatingDiffPackageForService -ArgumentList @($localServiceManifestName, $clusterServiceManifest.Version, $localServiceManifestVersion))
+
+            Copy-DiffPackage -clusterPackages $clusterServiceManifest.CodePackage -localPackages $localServiceManifest.CodePackage -localParentPkgPath $localServicePkgPath -diffParentPkgPath $diffServicePkgPath
+            Copy-DiffPackage -clusterPackages $clusterServiceManifest.ConfigPackage -localPackages $localServiceManifest.ConfigPackage -localParentPkgPath $localServicePkgPath -diffParentPkgPath $diffServicePkgPath
+            Copy-DiffPackage -clusterPackages $clusterServiceManifest.DataPackage -localPackages $localServiceManifest.DataPackage -localParentPkgPath $localServicePkgPath -diffParentPkgPath $diffServicePkgPath
+
+            Write-Host (Get-VstsLocString -Key DIFFPKG_CopyingToDiffPackge -ArgumentList @($localServiceManifestPath, (Join-Path $diffServicePkgPath $serviceManifestName)))
+            Copy-Item $localServiceManifestPath (Join-Path $diffServicePkgPath $serviceManifestName) -Force
+        }
+
+        Return $diffPackagePath
+
+    } finally {
+        Trace-VstsLeavingInvocation $MyInvocation
+    }
+}
+
+function Copy-DiffPackage
+{
+    param (
+        [array] $clusterPackages,
+        [array] $localPackages,
+        [string] $localParentPkgPath,
+        [string] $diffParentPkgPath
+    )
+
+    $clusterPackagesByName = @{}
+
+    foreach ($clusterPackage in $clusterPackages)
+    {
+        $clusterPackagesByName[$clusterPackage.Name] = $clusterPackage
+    }
+
+    foreach ($localPackage in $localPackages)
+    {
+        $clusterPackage = $clusterPackagesByName[$localPackage.Name]
+
+        # If cluster package exists and the version is the same to the local package version, do not add the local package to Diff Package
+        if ($clusterPackage.Version -eq $localPackage.Version)
+        {
+            continue
+        }
+
+        $localPkgPath = Join-Path $localParentPkgPath $localPackage.Name
+        $diffPkgPath = Join-Path $diffParentPkgPath $localPackage.Name
+
+        Write-Host (Get-VstsLocString -Key DIFFPKG_CopyingToDiffPackge -ArgumentList @($localPkgPath, $diffPkgPath))
+        # Copy the package on this level to diff package which is considered to be Leaf
+        Copy-Item $localPkgPath $diffPkgPath -Recurse
+    }
+    return
+}

Tasks/ServiceFabricDeploy/Strings/resources.resjson/en-US/resources.resjson

@@ -16,6 +16,8 @@
   "loc.input.help.applicationParameterPath": "Path to the application parameters file. [Variables](https://go.microsoft.com/fwlink/?LinkID=550988) and wildcards can be used in the path. If specified, this will override the value in the publish profile.",
   "loc.input.label.compressPackage": "Compress Package",
   "loc.input.help.compressPackage": "Indicates whether the application package should be compressed before copying to the image store. If enabled, this will override the value in the publish profile.",
+  "loc.input.label.useDiffPackage": "Use Diff Package",
+  "loc.input.help.useDiffPackage": "Upgrade by using a diff package that contains only the updated application files, the updated application manifest, and the service manifest files.",
   "loc.input.label.copyPackageTimeoutSec": "CopyPackageTimeoutSec",
   "loc.input.help.copyPackageTimeoutSec": "Timeout in seconds for copying application package to image store. If specified, this will override the value in the publish profile.",
   "loc.input.label.registerPackageTimeoutSec": "RegisterPackageTimeoutSec",
@@ -60,6 +62,14 @@
   "loc.input.help.registryPassword": "Password for the Docker registry. If the password is not encrypted, it is recommended that you use a custom release definition secret variable to store it.",
   "loc.input.label.passwordEncrypted": "Password Encrypted",
   "loc.input.help.passwordEncrypted": "It is recommended to encrypt your password using [Invoke-ServiceFabricEncryptText](https://docs.microsoft.com/en-us/azure/service-fabric/service-fabric-application-secret-management#encrypt-application-secrets). If you do not, and a certificate matching the Server Certificate Thumbprint in the Cluster Connection is installed on the build agent, it will be used to encrypt the password; otherwise an error will occur.",
+  "loc.messages.DIFFPKG_ApplicationDoesNotExist": "The application {0} to be upgraded by using diff package does not exist in the cluster {1}. Applying full application packge to do the deployment now...",
+  "loc.messages.DIFFPKG_ApplicationIsNotChanged": "The application {0} to be upgraded by using diff package has the same version {1} running in the cluster {2}. Skip upgrading it.",
+  "loc.messages.DIFFPKG_CopyingToDiffPackge": "Copying {0} from full application package to {1} in diff package...",
+  "loc.messages.DIFFPKG_CreatingDiffPackage": "Trying to create diff package...",
+  "loc.messages.DIFFPKG_CreatingDiffPackageForService": "Creating diff package for service: {0}, clusterServiceManifest.Version: {1}, localServiceManifest.Version: {2}",
+  "loc.messages.DIFFPKG_NoServicesRunning": "No services of application {0} are running in the cluster {1}. Applying full application packge to do the deployment now...",
+  "loc.messages.DIFFPKG_ServiceDoesNotExist": "The service {0} of application {1} to be upgraded by using diff package does not exist in the cluster {2}. Copying it to diff package now...",
+  "loc.messages.DIFFPKG_ServiceIsNotChanged": "The service {0} of application {1} to be upgraded by using diff package has the same version {2} running in the cluster {3}. Skip upgrading it.",
   "loc.messages.ItemSearchMoreThanOneFound": "Found more than one item with search pattern {0}. There can be only one.",
   "loc.messages.ItemSearchNoFilesFound": "No items were found with search pattern {0}.",
   "loc.messages.SearchingForPath": "Searching for path: {0}",

Tasks/ServiceFabricDeploy/Tests/CreateDiffPkg.ps1

@@ -0,0 +1,114 @@
+[CmdletBinding()]
+param()
+
+. $PSScriptRoot\..\..\..\Tests\lib\Initialize-Test.ps1
+
+$publishProfilePath = "$PSScriptRoot\data\NoAuthPublishProfile.xml"
+$applicationPackagePath = "$PSScriptRoot\data\DiffPkgAssets\AppPkg"
+$diffPackagePath = $env:TEMP + "\DiffPackage"
+$serviceConnectionName = "random connection name"
+$serviceFabricSdkModulePath = "$PSScriptRoot\data\ServiceFabricSDK.ps1"
+$appName = "AppName"
+$overwriteBehavior = "SameAppTypeAndVersion"
+$applicationTypeName = "TestType"
+$applicationTypeVersion = "1.0.0"
+$appManifestPath = "$PSScriptRoot\data\DiffPkgAssets\AppPkg\ApplicationManifest.xml"
+$appManifestDiffPath = $diffPackagePath + "\ApplicationManifest.xml"
+$serviceManifestPath1 = "$PSScriptRoot\data\DiffPkgAssets\AppPkg\Stateless1Pkg\ServiceManifest.xml"
+$serviceManifestDiffPath1 = $diffPackagePath + "\Stateless1Pkg\ServiceManifest.xml"
+$codePkg1 = "$PSScriptRoot\data\DiffPkgAssets\AppPkg\Stateless1Pkg\Code"
+$codeDiffPkg1 = $diffPackagePath + "\Stateless1Pkg\Code"
+$serviceManifestPath2 = "$PSScriptRoot\data\DiffPkgAssets\AppPkg\Stateless2Pkg\ServiceManifest.xml"
+$serviceManifestDiffPath2 = $diffPackagePath + "\Stateless2Pkg\ServiceManifest.xml"
+$codePkg2 = "$PSScriptRoot\data\DiffPkgAssets\AppPkg\Stateless2Pkg\Code"
+$codeDiffPkg2 = $diffPackagePath + "\Stateless2Pkg\Code"
+
+# Setup input arguments
+Register-Mock Get-VstsInput { $publishProfilePath } -- -Name publishProfilePath
+Register-Mock Get-VstsInput { $applicationPackagePath } -- -Name applicationPackagePath -Require
+Register-Mock Get-VstsInput { $serviceConnectionName } -- -Name serviceConnectionName -Require
+Register-Mock Get-VstsInput { "false" } -- -Name compressPackage
+Register-Mock Get-VstsInput { $overwriteBehavior } -- -Name overwriteBehavior
+Register-Mock Get-VstsInput { "false" } -- -Name skipUpgradeSameTypeAndVersion
+Register-Mock Get-VstsInput { "false" } -- -Name skipPackageValidation
+Register-Mock Get-VstsInput { "false" } -- -Name unregisterUnusedVersions
+Register-Mock Get-VstsInput { "false" } -- -Name configureDockerSettings
+Register-Mock Get-VstsInput { $true }  --  -Name useDiffPackage
+
+# Setup file resolution
+Register-Mock Find-VstsFiles { $publishProfilePath } -- -LegacyPattern $publishProfilePath
+Register-Mock Find-VstsFiles { $applicationPackagePath } -- -LegacyPattern $applicationPackagePath -IncludeDirectories
+
+Register-Mock Assert-VstsPath
+Register-Mock Test-Path { $true } -- "HKLM:\SOFTWARE\Microsoft\Service Fabric SDK"
+
+# Setup mock VSTS service endpoint
+$vstsEndpoint = @{
+    "Auth" = @{
+        "Scheme" = "None"
+    }
+}
+Register-Mock Get-VstsEndpoint { $vstsEndpoint } -- -Name $serviceConnectionName -Require
+
+# Setup mock for connection to cluster
+Register-Mock Connect-ServiceFabricCluster { $null } -- -ConnectionEndpoint "test"
+
+# Setup mock registry settings
+$regKeyObj = @{
+    "FabricSDKPSModulePath" = $serviceFabricSdkModulePath
+}
+Register-Mock Get-ItemProperty { $regKeyObj } -- -Path "HKLM:\SOFTWARE\Microsoft\Service Fabric SDK" -Name FabricSDKPSModulePath
+
+Register-Mock Get-ApplicationNameFromApplicationParameterFile { $appName } -- "$PSScriptRoot\data\ApplicationParameters.xml"
+
+$app = @{
+    "ApplicationTypeName" = $applicationTypeName;
+    "ApplicationTypeVersion" = $applicationTypeVersion
+}
+Register-Mock Get-ServiceFabricApplication { $app } -- -ApplicationName $appName
+$publishArgs = @("-ApplicationParameterFilePath:", "$PSScriptRoot\data\ApplicationParameters.xml", "-OverwriteBehavior:", $overwriteBehavior, "-ApplicationPackagePath:", $diffPackagePath, "-ErrorAction:", "Stop", "-Action:", "RegisterAndCreate")
+Register-Mock Publish-NewServiceFabricApplication -Arguments $publishArgs
+
+$serviceType1 = @{
+    "ServiceManifestName" = "Stateless1Pkg"
+}
+$serviceType2 = @{
+    "ServiceManifestName" = "Stateless2Pkg"
+}
+$serviceTypes = @($serviceType1, $serviceType2)
+$serviceManifest1 = '<ServiceManifest Name="Stateless1Pkg" Version="1.0.0">' +
+  '<ServiceTypes>' +
+    '<StatelessServiceType ServiceTypeName="Stateless1Type" />' +
+  '</ServiceTypes>' +
+  '<CodePackage Name="Code" Version="1.0.0">' +
+  '</CodePackage>' +
+  '<ConfigPackage Name="Config" Version="1.0.0" />' +
+'</ServiceManifest>'
+$serviceManifest2 = '<ServiceManifest Name="Stateless2Pkg" Version="1.0.0">' +
+  '<ServiceTypes>' +
+    '<StatelessServiceType ServiceTypeName="Stateless2Type" />' +
+  '</ServiceTypes>' +
+  '<CodePackage Name="Code" Version="1.0.0">' +
+  '</CodePackage>' +
+  '<ConfigPackage Name="Config" Version="1.0.0" />' +
+'</ServiceManifest>'
+
+Register-Mock Get-ServiceFabricServiceType {$serviceTypes} -- -ApplicationTypeName $applicationTypeName -ApplicationTypeVersion $applicationTypeVersion
+Register-Mock Get-ServiceFabricServiceManifest {$serviceManifest1} -- -ApplicationTypeName $applicationTypeName -ApplicationTypeVersion $applicationTypeVersion -ServiceManifestName "Stateless1Pkg"
+Register-Mock Get-ServiceFabricServiceManifest {$serviceManifest2} -- -ApplicationTypeName $applicationTypeName -ApplicationTypeVersion $applicationTypeVersion -ServiceManifestName "Stateless2Pkg"
+
+Register-Mock Copy-Item {} $appManifestPath $appManifestDiffPath -Force
+
+Microsoft.PowerShell.Core\Import-Module "$PSScriptRoot\..\Create-DiffPackage.psm1"
+
+# Act
+. $PSScriptRoot\..\..\..\Tasks\ServiceFabricDeploy\ps_modules\ServiceFabricHelpers\Connect-ServiceFabricClusterFromServiceEndpoint.ps1
+@( & $PSScriptRoot/../../../Tasks/ServiceFabricDeploy/deploy.ps1 )
+
+# Assert
+Assert-WasCalled Copy-Item $appManifestPath $appManifestDiffPath -Force
+Assert-WasCalled Copy-Item $serviceManifestPath1 $serviceManifestDiffPath1 -Force -Times 0
+Assert-WasCalled Copy-Item $serviceManifestPath2 $serviceManifestDiffPath2 -Force
+Assert-WasCalled Copy-Item $codePkg1 $codeDiffPkg1 -Recurse -Times 0
+Assert-WasCalled Copy-Item $codePkg2 $codeDiffPkg2 -Recurse
+Assert-WasCalled Publish-NewServiceFabricApplication -Arguments $publishArgs

Tasks/ServiceFabricDeploy/Tests/L0.ts

@@ -42,5 +42,8 @@ describe('ServiceFabricDeploy Suite', function () {
         it('Certificate deploy with Docker support', (done) => {
             psr.run(path.join(__dirname, 'CertDeployWithDocker.ps1'), done);
         })
+        it('Deploy with diff pkg', (done) => {
+            psr.run(path.join(__dirname, 'CreateDiffPkg.ps1'), done);
+        })
     }
 });

Tasks/ServiceFabricDeploy/Tests/data/DiffPkgAssets/AppPkg/ApplicationManifest.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="utf-8"?>
+<ApplicationManifest xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" ApplicationTypeName="TestType" ApplicationTypeVersion="2.0.0" xmlns="http://schemas.microsoft.com/2011/01/fabric">
+  <Parameters>
+    <Parameter Name="Stateless2_InstanceCount" DefaultValue="-1" />
+    <Parameter Name="Stateless1_InstanceCount" DefaultValue="-1" />
+  </Parameters>
+  <!-- Import the ServiceManifest from the ServicePackage. The ServiceManifestName and ServiceManifestVersion 
+       should match the Name and Version attributes of the ServiceManifest element defined in the 
+       ServiceManifest.xml file. -->
+  <ServiceManifestImport>
+    <ServiceManifestRef ServiceManifestName="Stateless2Pkg" ServiceManifestVersion="2.0.0" />
+    <ConfigOverrides />
+  </ServiceManifestImport>
+  <ServiceManifestImport>
+    <ServiceManifestRef ServiceManifestName="Stateless1Pkg" ServiceManifestVersion="1.0.0" />
+    <ConfigOverrides />
+  </ServiceManifestImport>
+  <DefaultServices>
+    <!-- The section below creates instances of service types, when an instance of this 
+         application type is created. You can also create one or more instances of service type using the 
+         ServiceFabric PowerShell module.
+         
+         The attribute ServiceTypeName below must match the name defined in the imported ServiceManifest.xml file. -->
+    <Service Name="Stateless2" ServicePackageActivationMode="ExclusiveProcess">
+      <StatelessService ServiceTypeName="Stateless2Type" InstanceCount="[Stateless2_InstanceCount]">
+        <SingletonPartition />
+      </StatelessService>
+    </Service>
+    <Service Name="Stateless1" ServicePackageActivationMode="ExclusiveProcess">
+      <StatelessService ServiceTypeName="Stateless1Type" InstanceCount="[Stateless1_InstanceCount]">
+        <SingletonPartition />
+      </StatelessService>
+    </Service>
+  </DefaultServices>
+</ApplicationManifest>