fix: Improved resiliency of removal logic

avm/utilities/pipelines/e2eValidation/resourceRemoval/Initialize-DeploymentRemoval.ps1

@@ -72,6 +72,7 @@ function Initialize-DeploymentRemoval {
         # The initial sequence is a general order-recommendation
         $RemoveFirstSequence = @(
             'Microsoft.Authorization/locks',
+            'Microsoft.VirtualMachineImages/imageTemplates', # Must be removed before their MSI & should be removed before its entities permissions are removed
             'Microsoft.Authorization/roleAssignments',
             'Microsoft.Insights/diagnosticSettings',
             'Microsoft.Network/privateEndpoints/privateDnsZoneGroups',
@@ -91,7 +92,6 @@ function Initialize-DeploymentRemoval {
             'Microsoft.MachineLearningServices/workspaces',
             'Microsoft.Compute/virtualMachines',
             'Microsoft.ContainerInstance/containerGroups' # Must be removed before their MSI
-            'Microsoft.VirtualMachineImages/imageTemplates', # Must be removed before their MSI
             'Microsoft.ManagedIdentity/userAssignedIdentities',
             'Microsoft.Databricks/workspaces'
             'Microsoft.Resources/resourceGroups'

avm/utilities/pipelines/e2eValidation/resourceRemoval/helper/Get-DeploymentTargetResourceList.ps1

@@ -1,4 +1,109 @@
 #region helper
+<#
+.SYNOPSIS
+Get all deployment operations at a given scope
+
+.DESCRIPTION
+Get all deployment oeprations at a given scope. By default, the results are filtered down to 'create' operations (i.e., excluding 'read' operations that would correspond to 'existing' resources).
+
+.PARAMETER Name
+Mandatory. The deployment name to search for
+
+.PARAMETER ResourceGroupName
+Optional. The name of the resource group for scope 'resourcegroup'. Relevant for resource-group-level deployments.
+
+.PARAMETER SubscriptionId
+Optional. The ID of the subscription to fetch deployments from. Relevant for subscription- & resource-group-level deployments.
+
+.PARAMETER ManagementGroupId
+Optional. The ID of the management group to fetch deployments from. Relevant for management-group-level deployments.
+
+.PARAMETER Scope
+Mandatory. The scope to search in
+
+.PARAMETER ProvisioningOperationsToInclude
+Optional. The provisioning operations to include in the result set. By default, only 'create' operations are included.
+
+.EXAMPLE
+Get-DeploymentOperationAtScope -Scope 'subscription' -Name 'v73rhp24d7jya-test-apvmiaiboaai'
+
+Get all deployment operations for a deployment with name 'v73rhp24d7jya-test-apvmiaiboaai' at scope 'subscription'
+
+.NOTES
+This function is a standin for the Get-AzDeploymentOperation cmdlet, which does not provide the ability to filter by provisioning operation.
+As such, it was also returning 'existing' resources (i.e., with provisioningOperation=Read).
+#>
+function Get-DeploymentOperationAtScope {
+
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)]
+        [Alias('DeploymentName')]
+        [string] $Name,
+
+        [Parameter(Mandatory = $false)]
+        [string] $ResourceGroupName,
+
+        [Parameter(Mandatory = $false)]
+        [string] $SubscriptionId,
+
+        [Parameter(Mandatory = $false)]
+        [string] $ManagementGroupId,
+
+        [Parameter(Mandatory = $false)]
+        [ValidateSet(
+            'Create', # any resource creation
+            'Read', # E.g., 'existing' resources
+            'EvaluateDeploymentOutput' # Nobody knows
+        )]
+        [string[]] $ProvisioningOperationsToInclude = @('Create'),
+
+        [Parameter(Mandatory)]
+        [ValidateSet(
+            'resourcegroup',
+            'subscription',
+            'managementgroup',
+            'tenant'
+        )]
+        [string] $Scope
+    )
+
+
+    switch ($Scope) {
+        'resourcegroup' {
+            $path = '/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Resources/deployments/{2}/operations?api-version=2021-04-01' -f $SubscriptionId, $ResourceGroupName, $name
+            break
+        }
+        'subscription' {
+            $path = '/subscriptions/{0}/providers/Microsoft.Resources/deployments/{1}/operations?api-version=2021-04-01' -f $SubscriptionId, $name
+            break
+        }
+        'managementgroup' {
+            $path = '/providers/Microsoft.Management/managementGroups/{0}/providers/Microsoft.Resources/deployments/{1}/operations?api-version=2021-04-01' -f $ManagementGroupId, $name
+            break
+        }
+        'tenant' {
+            $path = '/providers/Microsoft.Resources/deployments/{0}/operations?api-version=2021-04-01' -f $name
+            break
+        }
+    }
+
+    ##############################################
+    # Get all deployment children based on scope #
+    ##############################################
+
+    $response = Invoke-AzRestMethod -Method 'GET' -Path $path
+
+    if ($response.StatusCode -ne 200) {
+        Write-Error ('Failed to fetch deployment operations for deployment [{0}] in scope [{1}]' -f $name, $scope)
+        return
+    } else {
+        $deploymentOperations = ($response.content | ConvertFrom-Json).value.properties
+        $deploymentOperationsFiltered = $deploymentOperations | Where-Object { $_.provisioningOperation -in $ProvisioningOperationsToInclude }
+        return $deploymentOperationsFiltered ?? $true # Returning true to indicate that the deployment was found, but did not contain any relevant operations
+    }
+}
+
 <#
 .SYNOPSIS
 Get all deployments that match a given deployment name in a given scope
@@ -63,10 +168,18 @@ function Get-DeploymentTargetResourceListInner {
     ##############################################
     # Get all deployment children based on scope #
     ##############################################
+    $baseInputObject = @{
+        Scope          = $Scope
+        DeploymentName = $Name
+    }
     switch ($Scope) {
         'resourcegroup' {
             if (Get-AzResourceGroup -Name $resourceGroupName -ErrorAction 'SilentlyContinue') {
-                [array]$deploymentTargets = (Get-AzResourceGroupDeploymentOperation -DeploymentName $name -ResourceGroupName $resourceGroupName).TargetResource | Where-Object { $_ -ne $null } | Select-Object -Unique
+                if ($op = Get-DeploymentOperationAtScope @baseInputObject -ResourceGroupName $resourceGroupName -SubscriptionId $currentContext.Subscription.Id) {
+                    [array]$deploymentTargets = $op.TargetResource.id | Where-Object { $_ -ne $null } | Select-Object -Unique
+                } else {
+                    throw 'NoDeploymentFound'
+                }
             } else {
                 # In case the resource group itself was already deleted, there is no need to try and fetch deployments from it
                 # In case we already have any such resources in the list, we should remove them
@@ -75,15 +188,25 @@ function Get-DeploymentTargetResourceListInner {
             break
         }
         'subscription' {
-            [array]$deploymentTargets = (Get-AzDeploymentOperation -DeploymentName $name).TargetResource | Where-Object { $_ -ne $null } | Select-Object -Unique
+            if ($op = Get-DeploymentOperationAtScope @baseInputObject -SubscriptionId $currentContext.Subscription.Id) {
+                [array]$deploymentTargets = $op.TargetResource.id | Where-Object { $_ -ne $null } | Select-Object -Unique
+            } else {
+                throw 'NoDeploymentFound'
+            }
             break
         }
         'managementgroup' {
-            [array]$deploymentTargets = (Get-AzManagementGroupDeploymentOperation -DeploymentName $name -ManagementGroupId $ManagementGroupId).TargetResource | Where-Object { $_ -ne $null } | Select-Object -Unique
-            break
+            if ($op = Get-DeploymentOperationAtScope @baseInputObject -ManagementGroupId $ManagementGroupId) {
+                [array]$deploymentTargets = $op.TargetResource.id | Where-Object { $_ -ne $null } | Select-Object -Unique
+            }
+            throw 'NoDeploymentFound'
         }
         'tenant' {
-            [array]$deploymentTargets = (Get-AzTenantDeploymentOperation -DeploymentName $name).TargetResource | Where-Object { $_ -ne $null } | Select-Object -Unique
+            if ($op = Get-DeploymentOperationAtScope @baseInputObject) {
+                [array]$deploymentTargets = $op.TargetResource.id | Where-Object { $_ -ne $null } | Select-Object -Unique
+            } else {
+                throw 'NoDeploymentFound'
+            }
             break
         }
     }
@@ -235,30 +358,35 @@ function Get-DeploymentTargetResourceList {
             if (-not [String]::IsNullOrEmpty($ManagementGroupId)) {
                 $innerInputObject['ManagementGroupId'] = $ManagementGroupId
             }
-
-            [array]$targetResources = Get-DeploymentTargetResourceListInner @innerInputObject
-            if ($targetResources.Count -gt 0) {
-                Write-Verbose ('Found & resolved deployment [{0}]' -f $deploymentNameObject.Name) -Verbose
+            try {
+                $targetResources = Get-DeploymentTargetResourceListInner @innerInputObject
+                Write-Verbose ('Found & resolved deployment [{0}]. [{1}] resources found to remove.' -f $deploymentNameObject.Name, $targetResources.Count) -Verbose
                 $deploymentNameObject.Resolved = $true
                 $resourcesToRemove += $targetResources
+            } catch {
+                $remainingDeploymentNames = ($deploymentNameObjects | Where-Object { -not $_.Resolved }).Name
+                Write-Verbose ('No deployment found by name(s) [{0}] in scope [{1}]. Retrying in [{2}] seconds [{3}/{4}]' -f ($remainingDeploymentNames -join ', '), $scope, $searchRetryInterval, $searchRetryCount, $searchRetryLimit) -Verbose
+                Start-Sleep $searchRetryInterval
+                $searchRetryCount++
             }
         }
 
         # Break check
         if ($deploymentNameObjects.Resolved -notcontains $false) {
             break
         }
+    } while ($searchRetryCount -le $searchRetryLimit)
 
+    if ($searchRetryCount -gt $searchRetryLimit) {
         $remainingDeploymentNames = ($deploymentNameObjects | Where-Object { -not $_.Resolved }).Name
-        Write-Verbose ('No deployment found by name(s) [{0}] in scope [{1}]. Retrying in [{2}] seconds [{3}/{4}]' -f ($remainingDeploymentNames -join ', '), $scope, $searchRetryInterval, $searchRetryCount, $searchRetryLimit) -Verbose
-        Start-Sleep $searchRetryInterval
-        $searchRetryCount++
-    } while ($searchRetryCount -le $searchRetryLimit)
 
-    if (-not $resourcesToRemove) {
-        Write-Warning ('No deployment target resources found for [{0}]' -f ($DeploymentNames -join ', '))
-        return @()
+        # We don't want to outright throw an exception as we want to remove as many resources as possible before failing the script in the calling function
+        return @{
+            resolveError      = ('No deployment for the deployment name(s) [{0}] found' -f ($remainingDeploymentNames -join ', '))
+            resourcesToRemove = $resourcesToRemove
+        }
+    }
+    return @{
+        resourcesToRemove = $resourcesToRemove
     }
-
-    return $resourcesToRemove
 }

avm/utilities/pipelines/e2eValidation/resourceRemoval/helper/Remove-Deployment.ps1

@@ -100,17 +100,21 @@ function Remove-Deployment {
             if (-not [String]::IsNullOrEmpty($ManagementGroupId)) {
                 $deploymentsInputObject['ManagementGroupId'] = $ManagementGroupId
             }
-            $deployedTargetResources += Get-DeploymentTargetResourceList @deploymentsInputObject
 
-            if ($deployedTargetResources.Count -eq 0) {
-                throw 'No deployment target resources found.'
-            }
+            # In case the function also returns an error, we'll throw a corresponding exception at the end of this script (see below)
+            $resolveResult = Get-DeploymentTargetResourceList @deploymentsInputObject
+            $deployedTargetResources += $resolveResult.resourcesToRemove
         }
 
         [array] $deployedTargetResources = $deployedTargetResources | Select-Object -Unique
 
         Write-Verbose ('Total number of deployment target resources after fetching deployments [{0}]' -f $deployedTargetResources.Count) -Verbose
 
+        if (-not $deployedTargetResources) {
+            # Nothing to do
+            return
+        }
+
         # Pre-Filter & order items
         # ========================
         $rawTargetResourceIdsToRemove = $deployedTargetResources | Sort-Object -Culture 'en-US' -Property { $_.Split('/').Count } -Descending | Select-Object -Unique
@@ -121,10 +125,6 @@ function Remove-Deployment {
         [array] $resourcesToRemove = Get-ResourceIdsAsFormattedObjectList -ResourceIds $rawTargetResourceIdsToRemove
         Write-Verbose ('Total number of deployment target resources after formatting items [{0}]' -f $resourcesToRemove.Count) -Verbose
 
-        if ($resourcesToRemove.Count -eq 0) {
-            return
-        }
-
         # Filter resources
         # ================
 
@@ -172,6 +172,11 @@ function Remove-Deployment {
         } else {
             Write-Verbose 'Found [0] resources to remove'
         }
+
+        # In case any deployment was not resolved as planned we finally want to throw an exception to make this visible in the pipeline
+        if ($resolveResult.resolveError) {
+            throw ('The following error was thrown while resolving the original deployment names: [{0}]' -f $resolveResult.resolveError)
+        }
     }
 
     end {

avm/utilities/pipelines/e2eValidation/resourceRemoval/helper/Remove-ResourceList.ps1

@@ -36,7 +36,7 @@ function Remove-ResourceListInner {
 
         foreach ($resource in $resourcesToRemove) {
             $resourceName = Split-Path $resource.resourceId -Leaf
-            $alreadyProcessed = $processedResources.count -gt 0 ? (($processedResources | Where-Object { $resource.resourceId -like ('{0}*' -f $_) }).Count -gt 0) : $false
+            $alreadyProcessed = $processedResources.count -gt 0 ? (($processedResources | Where-Object { $resource.resourceId -like ('{0}/*' -f $_) }).Count -gt 0) : $false
 
             if ($alreadyProcessed) {
                 # Skipping
@@ -130,6 +130,6 @@ function Remove-ResourceList {
     if ($resourcesToRetry.Count -gt 0) {
         throw ('The removal failed for resources [{0}]' -f ((Split-Path $resourcesToRetry.resourceId -Leaf) -join ', '))
     } else {
-        Write-Verbose 'The removal completed successfully'
+        Write-Verbose ('The removal of the [{0}] completed successfully' -f $ResourcesToRemove.Count)
     }
 }