diff --git a/Scripts/ImageFactory/Scripts/CleanUpFactory.ps1 b/Scripts/ImageFactory/Scripts/CleanUpFactory.ps1 index c257893bb..c397a6480 100644 --- a/Scripts/ImageFactory/Scripts/CleanUpFactory.ps1 +++ b/Scripts/ImageFactory/Scripts/CleanUpFactory.ps1 @@ -1,8 +1,5 @@ param ( - [Parameter(Mandatory=$true, HelpMessage="The name of the DevTest Lab resource group")] - [string] $ResourceGroupName, - [Parameter(Mandatory=$true, HelpMessage="The name of the DevTest Lab to clean up")] [string] $DevTestLabName ) @@ -18,7 +15,7 @@ $deleteVmBlock = { Import-Module $modulePath LoadProfile Write-Output "##[section]Deleting VM: $vmName" - Remove-AzureRmResource -ResourceId $resourceId -ApiVersion 2017-04-26-preview -Force + Remove-AzureRmResource -ResourceId $resourceId -ApiVersion 2016-05-15 -Force Write-Output "##[section]Completed deleting $vmName" } @@ -57,7 +54,8 @@ foreach ($currentVm in $allVms){ } # Find any custom images that failed to provision and delete those -$bustedLabCustomImages = Get-AzureRmResource -ResourceName $DevTestLabName -ResourceGroupName $ResourceGroupName -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2017-04-26-preview' | Where-Object {($_.Properties.ProvisioningState -ne "Succeeded") -and ($_.Properties.ProvisioningState -ne "Creating")} +$ResourceGroupName = (Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $DevTestLabName}).ResourceGroupName +$bustedLabCustomImages = Get-AzureRmResource -ResourceName $DevTestLabName -ResourceGroupName $ResourceGroupName -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2016-05-15' | Where-Object {($_.Properties.ProvisioningState -ne "Succeeded") -and ($_.Properties.ProvisioningState -ne "Creating")} # Delete the custom images we found in the search above foreach ($imageToDelete in $bustedLabCustomImages) { @@ -66,15 +64,11 @@ foreach ($imageToDelete in $bustedLabCustomImages) { if($jobs.Count -ne 0) { - try{ - Write-Output "Waiting for VM Delete jobs to complete" - foreach ($job in $jobs){ - Receive-Job $job -Wait | Write-Output - } - } - finally{ - Remove-Job -Job $jobs + Write-Output "Waiting for VM Delete jobs to complete" + foreach ($job in $jobs){ + Receive-Job $job -Wait | Write-Output } + Remove-Job -Job $jobs } else { diff --git a/Scripts/ImageFactory/Scripts/CreateImageFromVHD.json b/Scripts/ImageFactory/Scripts/CreateImageFromVHD.json index 539351cec..73c2efce0 100644 --- a/Scripts/ImageFactory/Scripts/CreateImageFromVHD.json +++ b/Scripts/ImageFactory/Scripts/CreateImageFromVHD.json @@ -58,7 +58,7 @@ }, "resources": [ { - "apiVersion": "2017-04-26-preview", + "apiVersion": "2016-05-15", "name": "[variables('resourceName')]", "type": "Microsoft.DevTestLab/labs/customimages", "tags": { diff --git a/Scripts/ImageFactory/Scripts/DistributeImages.ps1 b/Scripts/ImageFactory/Scripts/DistributeImages.ps1 index 0fee1e051..c51931cfb 100644 --- a/Scripts/ImageFactory/Scripts/DistributeImages.ps1 +++ b/Scripts/ImageFactory/Scripts/DistributeImages.ps1 @@ -5,18 +5,15 @@ param [Parameter(Mandatory=$true, HelpMessage="The ID of the subscription containing the images")] [string] $SubscriptionId, - - [Parameter(Mandatory=$true, HelpMessage="The name of the resource group")] - [string] $ResourceGroupName, [Parameter(Mandatory=$true, HelpMessage="The name of the lab")] [string] $DevTestLabName, [Parameter(Mandatory=$true, HelpMessage="The number of script blocks we can run in parallel")] [int] $maxConcurrentJobs - ) +$ErrorActionPreference = 'Continue' #resolve any relative paths in ConfigurationLocation $ConfigurationLocation = (Resolve-Path $ConfigurationLocation).Path @@ -25,129 +22,65 @@ $modulePath = Join-Path $scriptFolder "DistributionHelpers.psm1" Import-Module $modulePath SelectSubscription $SubscriptionId -#get the list of labs from the json file -$labsList = Join-Path $ConfigurationLocation "Labs.json" -$labInfo = ConvertFrom-Json -InputObject (gc $labsList -Raw) -$labInfoCount = $labInfo.Labs.Length -Write-Output "Found $labInfoCount total labs" - -logMessageForUnusedImagePaths $labInfo.Labs $ConfigurationLocation - -#get the list of images -$sourceLabLocation = (Get-AzureRmResource -ResourceName $DevTestLabName -ResourceGroupName $ResourceGroupName -ResourceType 'Microsoft.DevTestLab/labs').Location -$labImages = Get-AzureRmResource -ResourceName $DevTestLabName -ResourceGroupName $ResourceGroupName -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2017-04-26-preview' | Where-Object {$_.Properties.ProvisioningState -eq 'Succeeded'} -$labImageCount = $labImages.Count -Write-Output "Found $labImageCount images in lab $DevTestLabName" +SaveProfile -$sourceImageInfos = @{} +$sourceLab = Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $DevTestLabName} +$labStorageInfo = GetLabStorageInfo $sourceLab +$sourceImageInfos = GetImageInfosForLab $DevTestLabName $thingsToCopy = New-Object System.Collections.ArrayList -#iterate through the images in our lab and gather their storage keys once so we don't need to repeatedly grab them for the source lab later -foreach ($image in $labImages) { - $foundTag = getTagValue $image 'ImagePath' - if(!$foundTag) { - continue; - } - - $sourceVHDLocation = $image.Properties.Vhd.ImageName - $uri = New-Object System.Uri($sourceVHDLocation) - $vhdFileName = [System.IO.Path]::GetFileName($sourceVHDLocation) - $osType = $image.Properties.Vhd.OsType - $sysPrep = $image.Properties.Vhd.SysPrep - $sourceStorageAccountName = $uri.Host.Split('.')[0] - $sourceStorageAcct = (Get-AzureRMStorageAccountKey -StorageAccountName $sourceStorageAccountName -ResourceGroupName $ResourceGroupName) - - # Azure Powershell version 1.3.2 or below - https://msdn.microsoft.com/en-us/library/mt607145.aspx - $sourceStorageAccountKey = $sourceStorageAcct.Key1 - if ($sourceStorageAccountKey -eq $null) { - # Azure Powershell version 1.4 or greater: - $sourceStorageAccountKey = $sourceStorageAcct.Value[0] - } +$labsList = Join-Path $ConfigurationLocation "Labs.json" +$labInfo = ConvertFrom-Json -InputObject (gc $labsList -Raw) +logMessageForUnusedImagePaths $labInfo.Labs $ConfigurationLocation - $imageInfo = @{ - vhdLocation = $sourceVHDLocation - storageAccountName = $sourceStorageAccountName - storageAccountKey = $sourceStorageAccountKey - fileName = $vhdFileName - osType = $osType - isVhdSysPrepped = $sysPrep - imageDescription = $image.Properties.Description.Replace("Golden Image: ", "") - } - $sourceImageInfos[$image.Name] = $imageInfo -} +Write-Output "Found $($labInfo.Labs.Length) target labs" +$sortedLabList = $labInfo.Labs | Sort-Object {$_.SubscriptionId} -foreach ($targetLab in $labInfo.Labs){ +foreach ($targetLabInfo in $sortedLabList){ - foreach ($image in $labImages) { - $imageName = $image.Name - $targetLabName = $targetLab.LabName - $copyToLab = ShouldCopyImageToLab $targetLab $image + foreach ($sourceImage in $sourceImageInfos) { + $targetLabName = $targetLabInfo.LabName + $copyToLab = ShouldCopyImageToLab $targetLabInfo $sourceImage.imagePath if($copyToLab -eq $true) { - Write-Output "Gathering data to copy $imageName to $targetLabName" - - $imagePathValue = getTagValue $image 'ImagePath' - if(!$imagePathValue) { - Write-Output "Ignoring $imageName because it has no ImagePath tag specified" - continue; + SelectSubscription $targetLabInfo.SubscriptionId + $targetLabRG = (Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $targetLabName}).ResourceGroupName + if(!$targetLabRG) + { + Write-Error ("Unable to find a lab named $targetLabName in subscription with id " + $targetLabInfo.SubscriptionId) } - - $targetImageName = $imageName - - SelectSubscription $targetLab.SubscriptionId - - $lab = Get-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' -ResourceName $targetLabName -ResourceGroupName $targetLab.ResourceGroup - - $existingTargetImage = Get-AzureRmResource -ResourceName $targetLabName -ResourceGroupName $targetLab.ResourceGroup -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2017-04-26-preview' | Where-Object {$_.Name -eq $targetImageName} + + $targetLab = Get-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' -ResourceName $targetLabName -ResourceGroupName $targetLabRG + $targetLabStorageInfo = GetLabStorageInfo $targetLab + + $existingTargetImage = Get-AzureRmResource -ResourceName $targetLabName -ResourceGroupName $targetLabRG -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2016-05-15' | Where-Object {$_.Name -eq $sourceImage.imageName} if($existingTargetImage){ - Write-Output "Not copying $imageName to $targetLabName because it already exists there as $targetImageName" + Write-Output "$($sourceImage.imageName) already exists in $targetLabName and will not be overwritten" continue; } - $targetLabLocation = $lab.Location - if($targetLabLocation -ne $sourceLabLocation){ - Write-Error "Lab location does not match. Source lab $DevTestLabName is in $sourceLabLocation and target lab $targetLabName is in $targetLabLocation" + if($targetLab.Location -ne $sourceLab.Location){ + Write-Error "Lab location does not match. Source lab $DevTestLabName is in $($sourceLab.Location) and target lab $targetLabName is in $($targetLab.Location)" continue; } - $targetStorageAccount = $lab.Properties.DefaultStorageAccount - $splitStorageAcct = $targetStorageAccount.Split('/') - $targetStorageAcctName = $splitStorageAcct[$splitStorageAcct.Length - 1] - - # Azure Powershell version 1.3.2 or below - https://msdn.microsoft.com/en-us/library/mt607145.aspx - $targetStorageKey = (Get-AzureRMStorageAccountKey -StorageAccountName $targetStorageAcctName -ResourceGroupName $lab.ResourceGroupName).Key1 - - if ($targetStorageKey -eq $null) { - # Azure Powershell version 1.4 or greater: - $targetStorageKey = (Get-AzureRMStorageAccountKey -StorageAccountName $targetStorageAcctName -ResourceGroupName $lab.ResourceGroupName).Value[0] - } - - $sourceObject = $sourceImageInfos[$imageName] - - #make sure the destination has a generatedvhds container - $destContext = New-AzureStorageContext -StorageAccountName $targetStorageAcctName -StorageAccountKey $targetStorageKey - $existingContainer = Get-AzureStorageContainer -Context $destContext -Name 'generatedvhds' -ErrorAction Ignore - if($existingContainer -eq $null) - { - Write-Output 'Creating the generatedvhds container in the target storage account' - New-AzureStorageContainer -Context $destContext -Name generatedvhds - } - + Write-Output "Gathering data to copy $($sourceImage.imagePath) to $targetLabName" $copyObject = @{ - imageName = $targetImageName - sourceVHDLocation = $sourceObject.vhdLocation - sourceStorageAccountName = $sourceObject.storageAccountName - sourceStorageAccountKey = $sourceObject.storageAccountKey + imageName = $sourceImage.imageName + imageDescription = $sourceImage.description + imagePath = $sourceImage.imagePath + osType = $sourceImage.osType + isVhdSysPrepped = $true + vhdFileName = $sourceImage.vhdFileName + sourceStorageAccountName = $labStorageInfo.storageAcctName + sourceStorageKey = $labStorageInfo.storageAcctKey + sourceResourceGroup = $labStorageInfo.resourceGroupName + sourceSubscriptionId = $SubscriptionId targetLabName = $targetLabName - targetStorageKey = $targetStorageKey - targetStorageAccountName = $targetStorageAcctName - targetResourceGroup = $targetLab.ResourceGroup - fileName = $sourceObject.fileName + targetStorageAccountName = $targetLabStorageInfo.storageAcctName + targetStorageKey = $targetLabStorageInfo.storageAcctKey + targetResourceGroup = $targetLabStorageInfo.resourceGroupName targetSubscriptionId = $targetLab.SubscriptionId - osType = $sourceObject.osType - isVhdSysPrepped = $sourceObject.isVhdSysPrepped - imageDescription = $sourceObject.imageDescription - imagePath = $imagePathValue } $thingsToCopy.Add($copyObject) | Out-Null } @@ -162,12 +95,14 @@ $copyVHDBlock = { Param($modulePath, $copyObject, $scriptFolder, $SubscriptionId) Import-Module $modulePath LoadProfile - - $srcContext = New-AzureStorageContext -StorageAccountName $copyObject.sourceStorageAccountName -StorageAccountKey $copyObject.sourceStorageAccountKey + + $srcContext = New-AzureStorageContext -StorageAccountName $copyObject.sourceStorageAccountName -StorageAccountKey $copyObject.sourceStorageKey + $srcURI = $srcContext.BlobEndPoint + "imagefactoryvhds/" + $copyObject.vhdFileName $destContext = New-AzureStorageContext -StorageAccountName $copyObject.targetStorageAccountName -StorageAccountKey $copyObject.targetStorageKey - $copyHandle = Start-AzureStorageBlobCopy -srcUri $copyObject.sourceVHDLocation -SrcContext $srcContext -DestContainer 'generatedvhds' -DestBlob $copyObject.fileName -DestContext $destContext -Force + New-AzureStorageContainer -Context $destContext -Name 'imagefactoryvhds' -ErrorAction Ignore + $copyHandle = Start-AzureStorageBlobCopy -srcUri $srcURI -SrcContext $srcContext -DestContainer 'imagefactoryvhds' -DestBlob $copyObject.vhdFileName -DestContext $destContext -Force - Write-Output ("Started copying " + $copyObject.fileName + " to " + $copyObject.targetStorageAccountName + " at " + (Get-Date -format "h:mm:ss tt")) + Write-Output ("Started copying " + $copyObject.vhdFileName + " to " + $copyObject.targetStorageAccountName + " at " + (Get-Date -format "h:mm:ss tt")) $copyStatus = $copyHandle | Get-AzureStorageBlobCopyState $statusCount = 0 @@ -186,23 +121,25 @@ $copyVHDBlock = { if($copyStatus.Status -eq "Success") { - Write-Output ($copyObject.fileName + " successfully copied to Lab " + $copyObject.targetLabName + " Deploying image template") $imageName = $copyObject.imageName + Write-Output ($copyObject.vhdFileName + " successfully copied to Lab " + $copyObject.targetLabName + ". Deploying image $imageName") #now that we have a VHD in the right storage account we need to create the actual image by deploying an ARM template $templatePath = Join-Path $scriptFolder "CreateImageFromVHD.json" - $vhdUri = $destContext.BlobEndPoint + "generatedvhds/" + $copyObject.fileName + $vhdUri = $destContext.BlobEndPoint + "imagefactoryvhds/" + $copyObject.vhdFileName SelectSubscription $copyObject.targetSubscriptionId - Write-Output "Creating Image $imageName from template" $imagePath = $copyObject.imagePath - $deployName = "Deploy-$imageName".Replace(" ", "").Replace(",", "") + $deployName = "Deploy-$imageName" $deployResult = New-AzureRmResourceGroupDeployment -Name $deployName -ResourceGroupName $copyObject.targetResourceGroup -TemplateFile $templatePath -existingLabName $copyObject.targetLabName -existingVhdUri $vhdUri -imageOsType $copyObject.osType -isVhdSysPrepped $copyObject.isVhdSysPrepped -imageName $copyObject.imageName -imageDescription $copyObject.imageDescription -imagePath $imagePath + #delete the deployment information so that we dont use up the total deployments for this resource group + Remove-AzureRmResourceGroupDeployment -ResourceGroupName $copyObject.targetResourceGroup -Name $deployName -ErrorAction SilentlyContinue | Out-Null + if($deployResult.ProvisioningState -eq "Succeeded"){ Write-Output "Successfully deployed image. Deleting copied VHD" - Remove-AzureStorageBlob -Context $destContext -Container 'generatedvhds' -Blob $copyObject.fileName + Remove-AzureStorageBlob -Context $destContext -Container 'imagefactoryvhds' -Blob $copyObject.vhdFileName Write-Output "Copied VHD deleted" } else { @@ -211,7 +148,15 @@ $copyVHDBlock = { } else { - Write-Error "finished without success" + if($copyStatus) + { + Write-Output $copyStatus + Write-Error ("Copy Status should be Success but is reported as " + $copyStatus.Status) + } + else + { + Write-Error "There is no copy status" + } } } @@ -229,21 +174,36 @@ foreach ($copyObject in $thingsToCopy){ $jobIndex++ Write-Output "Creating background task to distribute image $jobIndex of $copyCount" $jobs += Start-Job -ScriptBlock $copyVHDBlock -ArgumentList $modulePath, $copyObject, $scriptFolder, $SubscriptionId -} +} if($jobs.Count -ne 0) { - try{ - Write-Output "Waiting for Image replication jobs to complete" - foreach ($job in $jobs){ - Receive-Job $job -Wait | Write-Output - } - } - finally{ - Remove-Job -Job $jobs + Write-Output "Waiting for $($jobs.Count) Image replication jobs to complete" + foreach ($job in $jobs){ + Receive-Job $job -Wait | Write-Output } + Remove-Job -Job $jobs } else { Write-Output "No images to distribute" } + +foreach ($copyInfo in $thingsToCopy) +{ + SelectSubscription $copyInfo.targetSubscriptionId + #remove the root container from the target labs since we dont need it any more + $targetLab = Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $copyInfo.targetLabName} + $targetStorageInfo = GetLabStorageInfo $targetLab + $storageContext = New-AzureStorageContext -StorageAccountName $targetStorageInfo.storageAcctName -StorageAccountKey $targetStorageInfo.storageAcctKey + $rootContainerName = 'imagefactoryvhds' + $rootContainer = Get-AzureStorageContainer -Context $storageContext -Name $rootContainerName -ErrorAction Ignore + if($rootContainer -ne $null) + { + Write-Output "Deleting the $rootContainerName container in the target storage account" + Remove-AzureStorageContainer -Context $storageContext -Name $rootContainerName + } + +} + +Write-Output "Distribution of $($jobs.Count) images is complete" diff --git a/Scripts/ImageFactory/Scripts/DistributionHelpers.psm1 b/Scripts/ImageFactory/Scripts/DistributionHelpers.psm1 index 03531ddb7..ce7a48a5e 100644 --- a/Scripts/ImageFactory/Scripts/DistributionHelpers.psm1 +++ b/Scripts/ImageFactory/Scripts/DistributionHelpers.psm1 @@ -1,8 +1,8 @@ function SelectSubscription($subId){ # switch to another subscription assuming it's not the one we're already on - if((Get-AzureRmContext).Subscription.SubscriptionId -ne $subId){ + if((Get-AzureRmContext).Subscription.Id -ne $subId){ Write-Output "Switching to subscription $subId" - Select-AzureRmSubscription -SubscriptionId $subId | Out-Null + Set-AzureRmContext -SubscriptionId $subId | Out-Null } } @@ -20,21 +20,14 @@ function getTagValue($resource, $tagName){ $result } -function ShouldCopyImageToLab ($lab, $image) +function ShouldCopyImageToLab ($lab, $imagePathValue) { $retval = $false - $imagePathTag = getTagValue $image 'ImagePath' - if(!$imagePathTag) { - #this image does not have the ImagePath tag. Dont copy it - $retval = $false - } - else{ - foreach ($labImagePath in $lab.ImagePaths) { - if ($imagePathTag.StartsWith($labImagePath.Replace("/", "\"))) { - $retVal = $true; - break; - } + foreach ($labImagePath in $lab.ImagePaths) { + if ($imagePathValue.StartsWith($labImagePath.Replace("/", "\"))) { + $retVal = $true; + break; } } $retval @@ -78,17 +71,113 @@ function SaveProfile { Remove-Item $profilePath } - Save-AzureRmProfile -Path $profilePath + Save-AzureRmContext -Path $profilePath } function LoadProfile { $scriptFolder = Split-Path $Script:MyInvocation.MyCommand.Path - Select-AzureRmProfile -Path (Join-Path $scriptFolder "profile.json") | Out-Null + Import-AzureRmContext -Path (Join-Path $scriptFolder "profile.json") | Out-Null } function deleteImage ($resourceGroupName, $resourceName) -{ +{ Write-Output "##[section]Deleting Image: $resourceName" - Remove-AzureRmResource -ResourceName $resourceName -ResourceGroupName $resourceGroupName -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2017-04-26-preview' -Force + Remove-AzureRmResource -ResourceName $resourceName -ResourceGroupName $resourceGroupName -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2016-05-15' -Force Write-Output "##[section]Completed deleting $resourceName" } + +function GetImageName ($imagePathValue) +{ + $splitImagePath = $imagePathValue.Split('\') + if($splitImagePath.Length -eq 1){ + #the image is directly in the GoldenImages folder. Just use the file name as the image name. + $imagename = $splitImagePath[0] + } + else { + #this image is in a folder within GoldenImages. Name the image with set to the name of the folder that contains the image + $segmentCount = $splitImagePath.Length + $imagename = $splitImagePath[$segmentCount - 2] + "_" + $splitImagePath[$segmentCount - 1] + } + + #clean up some special characters in the image name and stamp it with todays date + $imagename = $imagename.Replace(".json", "").Replace(".", "_").Replace(" ", "-") + $imagename = $imagename + "-" + (Get-Date -Format 'MMM-d-yyyy') + return $imagename +} + +function GetLabStorageInfo ($lab) +{ + $labRgName= $lab.ResourceGroupName + $sourceLab = Get-AzureRmResource -ResourceName $lab.Name -ResourceGroupName $labRgName -ResourceType 'Microsoft.DevTestLab/labs' + $storageAcctValue = $sourceLab.Properties.artifactsStorageAccount + $storageAcctName = $storageAcctValue.Substring($storageAcctValue.LastIndexOf('/') + 1) + + $storageAcct = (Get-AzureRMStorageAccountKey -StorageAccountName $storageAcctName -ResourceGroupName $labRgName) + # Azure Powershell version 1.3.2 or below - https://msdn.microsoft.com/en-us/library/mt607145.aspx + $storageAcctKey = $storageAcct.Key1 + if ($storageAcctKey -eq $null) { + # Azure Powershell version 1.4 or greater: + $storageAcctKey = $storageAcct.Value[0] + } + $result = @{ + resourceGroupName = $labRgName + storageAcctName = $storageAcctName + storageAcctKey = $storageAcctKey + } + return $result +} + +function EnsureRootContainerExists ($labStorageInfo) +{ + $storageContext = New-AzureStorageContext -StorageAccountName $labStorageInfo.storageAcctName -StorageAccountKey $labStorageInfo.storageAcctKey + $rootContainerName = 'imagefactoryvhds' + $rootContainer = Get-AzureStorageContainer -Context $storageContext -Name $rootContainerName -ErrorAction Ignore + if($rootContainer -eq $null) + { + Write-Output "Creating the $rootContainerName container in the target storage account" + $rootContainer = New-AzureStorageContainer -Context $storageContext -Name $rootContainerName + } +} + +function GetImageInfosForLab ($DevTestLabName) +{ + $lab = Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $DevTestLabName} + $labRgName= $lab.ResourceGroupName + $sourceLab = Get-AzureRmResource -ResourceName $DevTestLabName -ResourceGroupName $labRgName -ResourceType 'Microsoft.DevTestLab/labs' + $storageAcctValue = $sourceLab.Properties.artifactsStorageAccount + $storageAcctName = $storageAcctValue.Substring($storageAcctValue.LastIndexOf('/') + 1) + + $storageAcct = (Get-AzureRMStorageAccountKey -StorageAccountName $storageAcctName -ResourceGroupName $labRgName) + # Azure Powershell version 1.3.2 or below - https://msdn.microsoft.com/en-us/library/mt607145.aspx + $storageAcctKey = $storageAcct.Key1 + if ($storageAcctKey -eq $null) { + # Azure Powershell version 1.4 or greater: + $storageAcctKey = $storageAcct.Value[0] + } + + $storageContext = New-AzureStorageContext -StorageAccountName $storageAcctName -StorageAccountKey $storageAcctKey + + $rootContainerName = 'imagefactoryvhds' + + $jsonBlobs = Get-AzureStorageBlob -Context $storageContext -Container $rootContainerName -Blob '*json' + + Write-Host "Downloading $($jsonBlobs.Length) json files from storage account" + $downloadFolder = Join-Path $env:TEMP 'ImageFactoryDownloads' + if(Test-Path -Path $downloadFolder) + { + Remove-Item $downloadFolder -Recurse | Out-Null + } + New-Item -Path $downloadFolder -ItemType Directory | Out-Null + $jsonBlobs | Get-AzureStorageBlobContent -Destination $downloadFolder | Out-Null + + $sourceImageInfos = @() + + $downloadedFileNames = Get-ChildItem -Path $downloadFolder + foreach($file in $downloadedFileNames) + { + $imageObj = (gc $file.FullName -Raw) | ConvertFrom-Json + $sourceImageInfos += $imageObj + } + + return $sourceImageInfos +} diff --git a/Scripts/ImageFactory/Scripts/MakeGoldenImageVMs.ps1 b/Scripts/ImageFactory/Scripts/MakeGoldenImageVMs.ps1 index ce7aff3e3..f64ca5728 100644 --- a/Scripts/ImageFactory/Scripts/MakeGoldenImageVMs.ps1 +++ b/Scripts/ImageFactory/Scripts/MakeGoldenImageVMs.ps1 @@ -3,9 +3,6 @@ [Parameter(Mandatory=$true, HelpMessage="The location of the factory configuration files")] [string] $ConfigurationLocation, - [Parameter(Mandatory=$true, HelpMessage="The name of the resource group")] - [string] $ResourceGroupName, - [Parameter(Mandatory=$true, HelpMessage="The name of the lab")] [string] $DevTestLabName, @@ -19,7 +16,10 @@ [int] $StandardTimeoutMinutes, [Parameter(Mandatory=$true, HelpMessage="The name of the lab")] - [string] $vmSize + [string] $vmSize, + + [Parameter(HelpMessage="Specifies whether or not to sysprep the created VMs")] + [boolean] $includeSysprep = $true ) @@ -39,10 +39,10 @@ function IsVirtualMachineReady ($vmName, $status) if ($status.Count -gt 1) { # We have both parameters (provisioning state + power state) - this is the default case - if (($status[0].Code -eq "ProvisioningState/succeeded") -and ($status[1].Code -eq "PowerState/running")) { + if (($status[0].Code -eq "ProvisioningState/succeeded") -and ($status[1].Code -eq "PowerState/deallocated")) { $retval = $true } - elseif (($status[1].Code -eq "ProvisioningState/succeeded") -and ($status[0].Code -eq "PowerState/running")) { + elseif (($status[1].Code -eq "ProvisioningState/succeeded") -and ($status[0].Code -eq "PowerState/deallocated")) { $retval = $true } } @@ -73,7 +73,14 @@ foreach ($file in $files) $imagePath = $file.FullName.Substring($imageListLocation.Length + 1) #determine a VM name for each file - $vmName = $file.BaseName.Replace("_", "").Replace(" ", ""); + $vmName = $file.BaseName.Replace("_", "").Replace(" ", "").Replace(".", "") + $intName = 0 + if ([System.Int32]::TryParse($vmName, [ref]$intName)) + { + Write-Output "Adding prefix to vm named $vmName because it cannot be fully numeric" + $vmName = ('vm' + $vmName) + } + if($vmName.Length -gt 15){ $shortenedName = $vmName.Substring(0, 13) Write-Output "VM name $vmName is too long. Shortening to $shortenedName" @@ -92,31 +99,27 @@ foreach ($file in $files) $usedVmNames += $vmName Write-Output "Starting job to create a VM named $vmName for $imagePath" - $jobs += Start-Job -Name $file.Name -FilePath $makeVmScriptLocation -ArgumentList $modulePath, $file.FullName, $ResourceGroupName, $DevTestLabName, $vmName, $imagePath, $machineUserName, $machinePassword, $vmSize + $jobs += Start-Job -Name $file.Name -FilePath $makeVmScriptLocation -ArgumentList $modulePath, $file.FullName, $DevTestLabName, $vmName, $imagePath, $machineUserName, $machinePassword, $vmSize, $includeSysprep } -try{ - $jobCount = $jobs.Count - Write-Output "Waiting for $jobCount VM creation jobs to complete" - foreach ($job in $jobs){ - $jobOutput = Receive-Job $job -Wait - Write-Output $jobOutput - $createdVMName = $jobOutput[$jobOutput.Length - 1] - if($createdVMName){ - $createdVms.Add($createdVMName) - } +$jobCount = $jobs.Count +Write-Output "Waiting for $jobCount VM creation jobs to complete" +foreach ($job in $jobs){ + $jobOutput = Receive-Job $job -Wait + Write-Output $jobOutput + $createdVMName = $jobOutput[$jobOutput.Length - 1] + if($createdVMName){ + $createdVms.Add($createdVMName) } } -finally{ - Remove-Job -Job $jobs -} +Remove-Job -Job $jobs #get machines that show up in the VM blade so we can apply the GoldenImage Tag $allVms = Find-AzureRmResource -ResourceType "Microsoft.Compute/virtualMachines" for ($index = 0; $index -lt $createdVms.Count; $index++){ $currentVmName = $createdVms[$index] - $currentVmValue = $allVms | Where-Object {$_.Name -eq $currentVmName -and $_.ResourceGroupName.StartsWith($DevTestLabName)} + $currentVmValue = $allVms | Where-Object {$_.Name -eq $currentVmName -and $_.ResourceGroupName.StartsWith($DevTestLabName, "CurrentCultureIgnoreCase")} if(!$currentVmValue){ Write-Error "##[error]$currentVmName was not created successfully. It does not appear in the VM blade" continue; diff --git a/Scripts/ImageFactory/Scripts/MakeVM.ps1 b/Scripts/ImageFactory/Scripts/MakeVM.ps1 index 68c90a141..59dec8466 100644 --- a/Scripts/ImageFactory/Scripts/MakeVM.ps1 +++ b/Scripts/ImageFactory/Scripts/MakeVM.ps1 @@ -6,9 +6,6 @@ [Parameter(Mandatory=$true, HelpMessage="The full path of the template file")] [string] $TemplateFilePath, - [Parameter(Mandatory=$true, HelpMessage="The name of the resource group")] - [string] $ResourceGroupName, - [Parameter(Mandatory=$true, HelpMessage="The name of the lab")] [string] $DevTestLabName, @@ -25,9 +22,53 @@ [System.Security.SecureString] $machinePassword, [Parameter(Mandatory=$true, HelpMessage="The name of the lab")] - [string] $vmSize + [string] $vmSize, + + [boolean] $includeSysprep = $false ) +function makeUpdatedTemplateFile ($origTemplateFile, $outputFile) +{ + $armTemplate= ConvertFrom-Json -InputObject (gc $origTemplateFile -Raw -Encoding Ascii) + + #add the Sysprep or deprovision artifact to the list of artifacts for the VM + $newArtifact = @{} + if ($armTemplate.resources[0].properties.galleryImageReference.osType -eq 'Windows') + { + $artifactName = 'windows-sysprep' + } + else + { + $artifactName = 'linux-deprovision' + } + + $fullArtifactId = "[resourceId('Microsoft.DevTestLab/labs/artifactSources/artifacts', parameters('labName'), 'public repo', '$artifactName')]" + $newArtifact.artifactId = $fullArtifactId + $existingArtifacts = $armTemplate.resources[0].properties.artifacts + if (!$existingArtifacts -or $existingArtifacts.Count -eq 0) + { + Write-Output "$origTemplateFile has no artifacts. Adding the $artifactName artifact" + $artifactCollection = New-Object System.Collections.ArrayList + $artifactCollection.Add($newArtifact) + $armTemplate.resources[0].properties | Add-Member -Type NoteProperty -name 'artifacts' -Value $artifactCollection -Force + } + elseif ($existingArtifacts[$existingArtifacts.count - 1].artifactId -eq $fullArtifactId) + { + Write-Output "$origTemplateFile already has the Sysprep/Deprovision artifact. It will not be added again" + } + else + { + #The ARM template does not end with the sysprep/deprovision artifact. We will add it + #this is the common case + Write-Output "Adding $artifactName artifact to $origTemplateFile template for deployment" + $armTemplate.resources[0].properties.artifacts += $newArtifact + } + + Write-Output "Writing modified ARM template to $outputFile" + ($armTemplate | ConvertTo-Json -Depth 100 | % { [System.Text.RegularExpressions.Regex]::Unescape($_) }).Replace('\', '\\') | Out-File $outputFile + +} + Import-Module $ModulePath LoadProfile @@ -41,20 +82,39 @@ if($existingVms.Count -ne 0){ return "" } else { - $vmDeployResult = New-AzureRmResourceGroupDeployment -Name "Deploy-$vmName" -ResourceGroupName $ResourceGroupName -TemplateFile $TemplateFilePath -labName $DevTestLabName -newVMName $vmName -userName $machineUserName -password $machinePassword -size $vmSize + $deployName = "Deploy-$vmName" + $ResourceGroupName = (Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $DevTestLabName}).ResourceGroupName + + if($includeSysprep) + { + $updatedTemplateFilePath = [System.IO.Path]::GetTempFileName() + makeUpdatedTemplateFile $TemplateFilePath $updatedTemplateFilePath + } + else + { + Write-Output "Skipping sysprep step" + $updatedTemplateFilePath = $TemplateFilePath + } - if($vmDeployResult.ProvisioningState -eq "Succeeded"){ - Write-Output "##[section]Successfully deployed $vmName from $imagePath" + $vmDeployResult = New-AzureRmResourceGroupDeployment -Name $deployName -ResourceGroupName $ResourceGroupName -TemplateFile $updatedTemplateFilePath -labName $DevTestLabName -newVMName $vmName -userName $machineUserName -password $machinePassword -size $vmSize + + #delete the deployment information so that we dont use up the total deployments for this resource group + Remove-AzureRmResourceGroupDeployment -ResourceGroupName $ResourceGroupName -Name $deployName -ErrorAction SilentlyContinue | Out-Null + if($vmDeployResult.ProvisioningState -eq "Succeeded"){ #set the imagePath tag on the VM Write-Output "Stamping the VM $vmName with originalImageFile $imagePath" $existingVm = Find-AzureRmResource -ResourceType "Microsoft.DevTestLab/labs/virtualMachines" -ResourceNameContains $DevTestLabName | Where-Object { $_.Name -eq "$DevTestLabName/$vmName"} #Determine if artifacts succeeded Write-Output "Determining artifact status." - $existingVmArtStatus = (Get-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs/virtualmachines' -Name $existingVm.Name -ResourceGroupName $existingVm.ResourceGroupName).Properties.ArtifactDeploymentStatus + $filter = '$expand=Properties($expand=ComputeVm,NetworkInterface,Artifacts)' + $vmResource = Get-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs/virtualmachines' -Name $existingVm.Name -ResourceGroupName $existingVm.ResourceGroupName -ODataQuery $filter + $existingVmArtStatus = $vmResource.Properties.ArtifactDeploymentStatus if ($existingVmArtStatus.totalArtifacts -eq 0 -or $existingVmArtStatus.deploymentStatus -eq "Succeeded") { + Write-Output "##[section]Successfully deployed $vmName from $imagePath" + $tags = $existingVm.Tags if((get-command -Name 'New-AzureRmResourceGroup').Parameters["Tag"].ParameterType.FullName -eq 'System.Collections.Hashtable'){ # Azure Powershell version 2.0.0 or greater - https://github.com/Azure/azure-powershell/blob/v2.0.1-August2016/documentation/release-notes/migration-guide.2.0.0.md#change-of-tag-parameters @@ -68,11 +128,41 @@ else { Write-Output "Getting resource ID from Existing Vm" $vmResourceId = $existingVm.ResourceId Write-Output "Resource ID: $vmResourceId" - Set-AzureRmResource -ResourceId $vmResourceId -Tag $tags -Force + Set-AzureRmResource -ResourceId $vmResourceId -Tag $tags -Force | Out-Null } else { - Write-Error "##[error]Deploying VM artifacts failed: $vmName from $TemplateFilePath" + if ($existingVmArtStatus.deploymentStatus -ne "Succeeded") + { + Write-Error ("##[error]Artifact deployment status is: " + $existingVmArtStatus.deploymentStatus) + } + Write-Error "##[error]Deploying VM artifacts failed. $vmName from $TemplateFilePath. Failure details follow:" + $failedArtifacts = ($vmResource.Properties.Artifacts | Where-Object {$_.status -ne "Succeeded"}) + if($failedArtifacts -ne $null) + { + foreach($failedArtifact in $failedArtifacts) + { + if($failedArtifact.status -eq 'Pending') + { + Write-Output ('Pending Artifact ID: ' + $failedArtifact.artifactId) + } + elseif($failedArtifact.status -eq 'Skipped') + { + Write-Output ('Skipped Artifact ID: ' + $failedArtifact.artifactId) + } + else + { + Write-Output ('Failed Artifact ID: ' + $failedArtifact.artifactId) + Write-Output (' Artifact Status: ' + $failedArtifact.status) + Write-Output (' DeploymentStatusMessage: ' + $failedArtifact.deploymentStatusMessage) + Write-Output (' VmExtensionStatusMessage: ' + $failedArtifact.vmExtensionStatusMessage) + Write-Output '' + } + } + } + + Write-Output "Deleting VM $vmName after failed artifact deployment" + Remove-AzureRmResource -ResourceId $existingVm.ResourceId -ApiVersion 2016-05-15 -Force } } else { diff --git a/Scripts/ImageFactory/Scripts/RetireImages.ps1 b/Scripts/ImageFactory/Scripts/RetireImages.ps1 index 82e8d005c..8990da931 100644 --- a/Scripts/ImageFactory/Scripts/RetireImages.ps1 +++ b/Scripts/ImageFactory/Scripts/RetireImages.ps1 @@ -5,9 +5,6 @@ param [Parameter(Mandatory=$true, HelpMessage="The ID of the subscription containing the Image Factory")] [string] $SubscriptionId, - - [Parameter(Mandatory=$true, HelpMessage="The name of the Image Factory resource group")] - [string] $ResourceGroupName, [Parameter(Mandatory=$true, HelpMessage="The name of the Image Factory DevTest Lab")] [string] $DevTestLabName, @@ -16,6 +13,41 @@ param [int] $ImagesToSave ) +function CleanFilesInLabStorageAccount($DevTestLabName, $ImagesToSave) +{ + $sourceImageInfos = GetImageInfosForLab $DevTestLabName + + $thingsToDelete = $sourceImageInfos | Group-Object {$_.imagePath} | + ForEach-Object {$_.Group | + Sort-Object timestamp -Descending | + Select-Object -Skip $ImagesToSave} + + if($thingsToDelete -and $thingsToDelete.Count -gt 0) + { + Write-Output "Found $($thingsToDelete.Count) ImageInfos to delete in the $storageAcctName storage account" + $rootContainerName = 'imagefactoryvhds' + $sourceLab = Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $DevTestLabName} + $labStorageInfo = GetLabStorageInfo $sourceLab + $storageContext = New-AzureStorageContext -StorageAccountName $labStorageInfo.storageAcctName -StorageAccountKey $labStorageInfo.storageAcctKey + + foreach($thingToDelete in $thingsToDelete) + { + $vhdBlobName = $thingToDelete.vhdFileName + Write-Output "deleting $vhdBlobName from $storageAcctName" + Remove-AzureStorageBlob -Context $storageContext -Container $rootContainerName -Blob $vhdBlobName -Force + + $jsonBlobName = $vhdBlobName.Replace('.vhd', '.json') + Write-Output "deleting $jsonBlobName from $storageAcctName" + Remove-AzureStorageBlob -Context $storageContext -Container $rootContainerName -Blob $jsonBlobName -Force + } + } + else + { + Write-Output "No files to delete in the $storageAcctName storage account" + } +} + + #resolve any relative paths in ConfigurationLocation $ConfigurationLocation = (Resolve-Path $ConfigurationLocation).Path @@ -23,6 +55,8 @@ $modulePath = Join-Path (Split-Path ($Script:MyInvocation.MyCommand.Path)) "Dist Import-Module $modulePath SaveProfile +CleanFilesInLabStorageAccount $DevTestLabName $ImagesToSave + $jobs = @() # Script block for deleting images @@ -38,6 +72,7 @@ $deleteImageBlock = { # Get the list of labs that we have distributed images to $labsList = Join-Path $ConfigurationLocation "Labs.json" $labInfo = ConvertFrom-Json -InputObject (gc $labsList -Raw) +$ResourceGroupName = (Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $DevTestLabName}).ResourceGroupName # Add our 'current' lab (the factory lab) to the list of labs we're going to iterate through $factorylabInfo = (New-Object PSObject | @@ -48,13 +83,15 @@ $factorylabInfo = (New-Object PSObject | $labInfo.Labs = ($labInfo.Labs + $factorylabInfo) $labInfo | ConvertTo-Json | Write-Output +$sortedLabList = $labInfo.Labs | Sort-Object {$_.SubscriptionId} # Iterate through all the labs -foreach ($selectedLab in $labInfo.Labs){ +foreach ($selectedLab in $sortedLabList){ # Get the list of images in the current lab SelectSubscription $selectedLab.SubscriptionId - $allImages = Get-AzureRmResource -ResourceName $selectedLab.LabName -ResourceGroupName $selectedLab.ResourceGroup -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2017-04-26-preview' + $selectedLabRG = (Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $selectedLab.LabName}).ResourceGroupName + $allImages = Get-AzureRmResource -ResourceName $selectedLab.LabName -ResourceGroupName $selectedLabRG -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2016-05-15' # Get the images to delete (generated by factory + only old images for each group based on retension policy) $imageObjectsToDelete = $allImages | ?{$_.Tags } | ForEach-Object { New-Object -TypeName PSObject -Prop @{ ResourceName=$_.ResourceName @@ -96,7 +133,7 @@ foreach ($selectedLab in $labInfo.Labs){ #if this is an image from a target lab, make sure it has not been removed from the labs.json file $labName = $selectedLab.LabName if($labName -ne $DevTestLabName){ - $shouldCopyToLab = ShouldCopyImageToLab -lab $selectedLab -image $image + $shouldCopyToLab = ShouldCopyImageToLab -lab $selectedLab -image $imagePath if(!$shouldCopyToLab){ Write-Output "Image $resName is has been removed from Labs.json for $labName. Starting job to remove the image." $jobs += Start-Job -Name $image.ResourceName -ScriptBlock $deleteImageBlock -ArgumentList $modulePath, $image @@ -113,15 +150,11 @@ foreach ($selectedLab in $labInfo.Labs){ if($jobs.Count -ne 0) { - try{ - Write-Output "Waiting for Image deletion jobs to complete" - foreach ($job in $jobs){ - Receive-Job $job -Wait | Write-Output - } - } - finally{ - Remove-Job -Job $jobs + Write-Output "Waiting for Image deletion jobs to complete" + foreach ($job in $jobs){ + Receive-Job $job -Wait | Write-Output } + Remove-Job -Job $jobs } else { diff --git a/Scripts/ImageFactory/Scripts/SnapImageFromVM.json b/Scripts/ImageFactory/Scripts/SnapImageFromVM.json deleted file mode 100644 index d1aecc75c..000000000 --- a/Scripts/ImageFactory/Scripts/SnapImageFromVM.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "existingLabName": { - "type": "string", - "metadata": { - "description": "Name of an existing lab where the custom image will be created." - } - }, - "existingVMResourceId": { - "type": "string", - "metadata": { - "description": "Resource ID of an existing VM from which the custom image will be created." - } - }, - "imageName": { - "type": "string", - "metadata": { - "description": "Name of the custom image being created or updated." - } - }, - "imageDescription": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Details about the custom image being created or updated." - } - }, - "osType": { - "type": "string", - "allowedValues": [ - "Linux", - "Windows" - ], - "metadata": { - "description": "The OS Type. Can either be Windows or Linux." - } - }, - "imagePath": { - "type": "string", - "metadata": { - "description": "The path to the golden image template" - } - } - }, - "variables": { - "resourceName": "[concat(parameters('existingLabName'), '/', parameters('imageName'))]", - "resourceType": "Microsoft.DevTestLab/labs/customimages", - "linuxOsInfo": { - "Linux": { - "linuxOsState": "DeprovisionRequested" - }, - "Windows": null - }, - "windowsOsInfo": { - "Linux": null, - "Windows": { - "windowsOsState": "SysprepRequested" - } - } - }, - "resources": [ - { - "apiVersion": "2017-04-26-preview", - "name": "[variables('resourceName')]", - "type": "Microsoft.DevTestLab/labs/customimages", - "tags": { - "ImagePath": "[parameters('imagePath')]" - }, - "properties": { - "author": "Image Factory", - "description": "[parameters('imageDescription')]", - "osType": "[parameters('osType')]", - "vm": { - "sourceVmId": "[parameters('existingVMResourceId')]", - "windowsOsInfo": "[variables('windowsOsInfo')[parameters('osType')]]", - "linuxOsInfo": "[variables('linuxOsInfo')[parameters('osType')]]" - } - } - } - ], - "outputs": { - "customImageId": { - "type": "string", - "value": "[resourceId(variables('resourceType'), parameters('existingLabName'), parameters('imageName'))]" - } - } -} diff --git a/Scripts/ImageFactory/Scripts/SnapImagesFromVMs.ps1 b/Scripts/ImageFactory/Scripts/SnapImagesFromVMs.ps1 index 3caf99c2f..6cf6e536c 100644 --- a/Scripts/ImageFactory/Scripts/SnapImagesFromVMs.ps1 +++ b/Scripts/ImageFactory/Scripts/SnapImagesFromVMs.ps1 @@ -1,117 +1,203 @@ param ( [Parameter(Mandatory=$true, HelpMessage="The name of the DevTest Lab")] - [string] $DevTestLabName, - - [Parameter(Mandatory=$true, HelpMessage="The name of the Resource Group that holds the DevTest Lab")] - [string] $ResourceGroupName + [string] $DevTestLabName ) +$ErrorActionPreference = "Stop" $modulePath = Join-Path (Split-Path ($Script:MyInvocation.MyCommand.Path)) "DistributionHelpers.psm1" Import-Module $modulePath SaveProfile +$lab = Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $DevTestLabName} +$labRgName= $lab.ResourceGroupName +$labStorageInfo = GetLabStorageInfo $lab +EnsureRootContainerExists $labStorageInfo +$existingImageInfos = GetImageInfosForLab $DevTestLabName + +$labVMs = Get-AzureRmResource -ResourceName $DevTestLabName -ResourceGroupName $labRgName -ResourceType 'Microsoft.DevTestLab/labs/virtualMachines' -ApiVersion '2016-05-15' | Where-Object {$_.Properties.ProvisioningState -eq 'Succeeded'} $jobs = @() +$copyObjects = New-Object System.Collections.ArrayList +try +{ -# Script block for deleting images -$createImageBlock = { - Param($modulePath, $imageToCreate) - Import-Module $modulePath - LoadProfile - - $imageName = $imageToCreate.imagename - $deployName = "Deploy-$imagename".Replace(" ", "").Replace(",", "") - Write-Output "Creating Image $imagename from template" - $deployResult = New-AzureRmResourceGroupDeployment -Name $deployName -ResourceGroupName $imageToCreate.ResourceGroupName -TemplateFile $imageToCreate.templatePath -existingLabName $imageToCreate.DevTestLabName -existingVMResourceId $imageToCreate.vmResourceId -imageName $imagename -imageDescription $imageToCreate.imageDescription -imagePath $imageToCreate.imagePath -osType $imageToCreate.osType - - if($deployResult.ProvisioningState -eq "Succeeded"){ - Write-Output "Successfully deployed image" - $foundimage = (Get-AzureRmResource -ResourceName $imageToCreate.DevTestLabName -ResourceGroupName $imageToCreate.ResourceGroupName -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2017-04-26-preview') | Where-Object {$_.name -eq $imagename} - if($foundimage.Count -eq 0){ - Write-Warning "$imagename was not created successfully" + foreach($labVm in $labVMs) + { + #make sure we have a container in the storage account that matches the imagepath and date for this vhd + $imagePath = getTagValue $labVm 'ImagePath' + if(!$imagePath) + { + Write-Output "Ignoring $($labVm.Name) because it does not have the ImagePath tag" + continue } - } - - if($deploySuccess -eq $false){ - Write-Error "Creation of Image $imageName failed" - } -} + $imageName = GetImageName $imagePath + while ($existingImageInfos | Where-Object {$_.imageName -eq $imageName}) + { + #There is an existing image with this name. We must be running the factory multiple times today. + $lastChar = $imageName[$imageName.Length - 1] + $intVal = 0 + if ([System.Int32]::TryParse($vmName, [ref]$intVal)) + { + #last character is a number (probably part of the date). Append an A + $imageName = $imageName + 'A' + } + else + { + #last character is a letter. Increment the letter + $newLastChar = [char](([int]$lastChar) + 1) + $imageName = $imageName.SubString(0, ($imageName.Length - 1)) + $newLastChar + } + } -# Get a pointer to all the VMs in the subscription -$allVms = Get-AzureRmResource -ResourceGroupName $ResourceGroupName -ResourceType Microsoft.DevTestLab/labs/virtualmachines -ResourceName $DevTestLabName -ApiVersion 2017-04-26-preview - -foreach ($currentVm in $allVms){ - #vms with the ImagePath tag are the ones we care about - $imagePathValue = getTagValue $currentVm 'ImagePath' + $fileId = ([Guid]::NewGuid()).ToString() - if($imagePathValue) { - Write-Output ("##[command] Found Virtual Machine Running, will snap image of " + $currentVm.Name) + $computeVM = Get-AzureRmVM -Status | Where-Object -FilterScript {$_.Id -eq $labVM.Properties.computeId} + + if(!$computeVM) + { + Write-Error ("Didnt find a compute VM with ID " + $labVM.Properties.computeId) + } - $splitImagePath = $imagePathValue.Split('\') - if($splitImagePath.Length -eq 1){ - #the image is directly in the GoldenImages folder. Just use the file name as the image name. - $newimagename = $splitImagePath[0] + #If the VM is still running that means it hasnt been sysprepped. dont try to copy the VHD because it will be locked. + if($computeVM.PowerState -and $computeVM.PowerState -eq 'VM deallocated') + { + $isReady = $true } - else { - #this image is in a folder within GoldenImages. Name the image with set to the name of the folder that contains the image - $segmentCount = $splitImagePath.Length - $newimagename = $splitImagePath[$segmentCount - 2] + " " + $splitImagePath[$segmentCount - 1] + else + { + $foundPowerState = $computeVM.Statuses | Where-Object {$_.Code -eq 'PowerState/deallocated'} + if($foundPowerState) + { + $isReady = $true + } + else + { + $isReady = $false + } } - #clean up some special characters in the image name and stamp it with todays date - $newimagename = $newimagename.Replace(".json", "").Replace(".", "_") - $newimagename = $newimagename + " (" + (Get-Date -Format 'MMM d, yyyy').ToString() + ")" + if($isReady -ne $true) + { + Write-Output ("$($labVM.Name) because it is not currently stopped/deallocated so it will not be copied") + continue + } - $scriptFolder = Split-Path $Script:MyInvocation.MyCommand.Path - $templatePath = Join-Path $scriptFolder "SnapImageFromVM.json" - - if($currentVm.Properties.OsType -eq "Windows") { - $osType = "Windows" + #get a SAS token that's good for the next four hours. that should be enough time to complete all the disk copy jobs. + Write-Output "Getting SAS token for disk $($computeVM.StorageProfile.OsDisk.Name) in resource group $($computeVM.ResourceGroupName)" + $mdiskURL = (Grant-AzureRmDiskAccess -ResourceGroupName $computeVM.ResourceGroupName -DiskName $computeVM.StorageProfile.OsDisk.Name -Access Read -DurationInSecond 14400).AccessSAS + if($mdiskURL -ne $null) + { + $copyInfo = @{ + computeRGName = $computeVM.ResourceGroupName + computeDiskname = $computeVM.StorageProfile.OsDisk.Name + sourceSASToken = $mdiskURL + osType = $labVM.Properties.osType + fileId = $fileId + description = $labVM.Properties.notes.Replace("Golden Image: ", "") + storageAcctName = $labStorageInfo.storageAcctName + storageAcctKey = $labStorageInfo.storageAcctKey + imagePath = $imagePath + imageName = $imageName + } + + $copyObjects.Add($copyInfo) } - else { - $osType = "Linux" + else + { + Write-Error "Unable to get SAS token for disk $($computeVM.StorageProfile.OsDisk.Name) in resource group $($computeVM.ResourceGroupName)" } + } - $imageToCreate = @{ - ImageName = $newimagename - ResourceGroupName = $ResourceGroupName - DevTestLabName = $DevTestLabName - templatePath = $templatePath - vmResourceId = $currentVm.ResourceId - imageDescription = $currentVm.Properties.Notes - imagePath = $imagePathValue - osType = $osType + $storeVHDBlock = { + Param($modulePath, $copyObject) + Import-Module $modulePath + LoadProfile + + $vhdFileName = $copyObject.fileId + ".vhd" + $jsonFileName = $copyObject.fileId + ".json" + $jsonFilePath = Join-Path $env:TEMP $jsonFileName + $imageName = $copyObject.imageName + Write-Output "Storing image: $imageName" + $vhdInfo = @{ + imageName = $imageName + imagePath = $copyObject.imagePath + description = $copyObject.description + osType = $copyObject.osType + vhdFileName = $vhdFileName + timestamp = (Get-Date).ToUniversalTime().ToString() } + + ConvertTo-Json -InputObject $vhdInfo | Out-File $jsonFilePath + + $storageContext = New-AzureStorageContext -StorageAccountName $copyObject.storageAcctName -StorageAccountKey $copyObject.storageAcctKey - $existingImage = Get-AzureRmResource -ResourceName $DevTestLabName -ResourceGroupName $ResourceGroupName -ResourceType 'Microsoft.DevTestLab/labs/customImages' -ApiVersion '2017-04-26-preview' | Where-Object -FilterScript {$_.Name -eq $newImageName} - if($existingImage){ - Write-Output "Skipping the creation of $newImageName becuse it already exists" + Set-AzureStorageBlobContent -Context $storageContext -File $jsonFilePath -Container 'imagefactoryvhds' + + Write-Output "Starting vhd copy..." + $copyHandle = Start-AzureStorageBlobCopy -AbsoluteUri $copyObject.sourceSASToken -DestContainer 'imagefactoryvhds' -DestBlob $vhdFileName -DestContext $storageContext -Force + + Write-Output ("Started copy of " + $copyObject.computeDiskname + " at " + (Get-Date -format "h:mm:ss tt")) + $copyStatus = $copyHandle | Get-AzureStorageBlobCopyState + $statusCount = 0 + + While($copyStatus.Status -eq "Pending"){ + $copyStatus = $copyHandle | Get-AzureStorageBlobCopyState + [int]$perComplete = ($copyStatus.BytesCopied/$copyStatus.TotalBytes)*100 + Write-Progress -Activity "Copying blob..." -status "Percentage Complete" -percentComplete "$perComplete" + + if($perComplete -gt $statusCount){ + $statusCount = [math]::Ceiling($perComplete) + 3 + Write-Output "%$perComplete percent complete" + } + + Start-Sleep 45 } - else{ - Write-Output "Starting job to create image $newimagename" - $jobs += Start-Job -Name $imageToCreate.ImageName -ScriptBlock $createImageBlock -ArgumentList $modulePath, $imageToCreate + if($copyStatus.Status -eq "Success") + { + Write-Output ("Successfully copied " + $copyObject.computeDiskname + " at " + (Get-Date -format "h:mm:ss tt")) + } + else + { + if($copyStatus) + { + Write-Output $copyStatus + Write-Error ("Copy Status should be Success but is reported as " + $copyStatus.Status) + } + else + { + Write-Error "There is no copy status" + } } } -} -if($jobs.Count -ne 0) -{ - try{ - $jobCount = $jobs.Count - Write-Output "Waiting for $jobCount Image creation jobs to complete" + foreach ($copyObject in $copyObjects){ + $jobIndex++ + Write-Output "Creating background task to store VHD $jobIndex of $($copyObjects.Count)" + $jobs += Start-Job -ScriptBlock $storeVHDBlock -ArgumentList $modulePath, $copyObject + } + + if($jobs.Count -ne 0) + { + Write-Output "Waiting for VHD replication jobs to complete" foreach ($job in $jobs){ Receive-Job $job -Wait | Write-Output } - } - finally{ Remove-Job -Job $jobs } - - Write-Output "Completed snapping images!" + else + { + Write-Output "No VHDs to replicate" + } } -else +finally { - Write-Output "No images to create!" + foreach ($copyObject in $copyObjects) + { + Write-Output "Reverting lock on disk $($copyObject.computeDiskname) in resource group $($copyObject.computeRGName)" + Revoke-AzureRmDiskAccess -ResourceGroupName $copyObject.computeRGName -DiskName $copyObject.computeDiskname + + } } +Write-Output 'Finished storing sysprepped VHDs' \ No newline at end of file diff --git a/Scripts/ImageFactory/Scripts/UpdateBuildAgent.ps1 b/Scripts/ImageFactory/Scripts/UpdateBuildAgent.ps1 index daeab96d0..79dd42654 100644 --- a/Scripts/ImageFactory/Scripts/UpdateBuildAgent.ps1 +++ b/Scripts/ImageFactory/Scripts/UpdateBuildAgent.ps1 @@ -2,9 +2,6 @@ ( [Parameter(Mandatory=$true, HelpMessage="The ID of the subscription containing the Image Factory")] [string] $SubscriptionId, - - [Parameter(Mandatory=$true, HelpMessage="The name of the Image Factory resource group")] - [string] $ResourceGroupName, [Parameter(Mandatory=$true, HelpMessage="The name of the Image Factory DevTest Lab")] [string] $DevTestLabName, @@ -18,12 +15,13 @@ ) # find the build agent in the subscription -$agentVM = Get-AzureRmResource -ResourceGroupName $ResourceGroupName -ResourceType Microsoft.DevTestLab/labs/virtualmachines -ResourceName $DevTestLabName -ApiVersion 2017-04-26-preview | Where-Object {$_.Name -eq $BuildAgent} +$ResourceGroupName = (Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $DevTestLabName}).ResourceGroupName +$agentVM = Get-AzureRmResource -ResourceGroupName $ResourceGroupName -ResourceType Microsoft.DevTestLab/labs/virtualmachines -ResourceName $DevTestLabName -ApiVersion 2016-05-15 | Where-Object {$_.Name -eq $BuildAgent} if ($agentVM -ne $null) { # Update the agent via DevTest Labs with the specified action (start or stop) - $status = Invoke-AzureRmResourceAction -ResourceGroupName $ResourceGroupName -ResourceType Microsoft.DevTestLab/labs/virtualmachines -ResourceName ($DevTestLabName + "/" + $BuildAgent) -Action $Action -ApiVersion 2017-04-26-preview -Force + $status = Invoke-AzureRmResourceAction -ResourceGroupName $ResourceGroupName -ResourceType Microsoft.DevTestLab/labs/virtualmachines -ResourceName ($DevTestLabName + "/" + $BuildAgent) -Action $Action -ApiVersion 2016-05-15 -Force if ($status.Status -eq 'Succeeded') { Write-Output "##[section] Successfully updated VSTS Build Agent: $BuildAgent , Action: $Action" diff --git a/Scripts/ImageFactory/Scripts/ValidateLabsJson.ps1 b/Scripts/ImageFactory/Scripts/ValidateLabsJson.ps1 new file mode 100644 index 000000000..9f6d16fbe --- /dev/null +++ b/Scripts/ImageFactory/Scripts/ValidateLabsJson.ps1 @@ -0,0 +1,52 @@ +param +( + [Parameter(Mandatory=$true, HelpMessage="The location of the factory configuration files")] + [string] $ConfigurationLocation +) + +#resolve any relative paths in ConfigurationLocation +$ConfigurationLocation = (Resolve-Path $ConfigurationLocation).Path + +$modulePath = Join-Path (Split-Path ($Script:MyInvocation.MyCommand.Path)) "DistributionHelpers.psm1" +Import-Module $modulePath + +$labsList = Join-Path $ConfigurationLocation "Labs.json" +$labInfo = ConvertFrom-Json -InputObject (gc $labsList -Raw) +$sortedLabList = $labInfo.Labs | Sort-Object {$_.SubscriptionId} +Write-Output "Validating $($sortedLabList.Count) labs from Labs.json" + + +$sortedLabList | Group-Object -Property SubscriptionId,LabName | Where-Object {$_.Count -gt 1} | + ForEach-Object { + $labName = $_.Group[0].LabName + $subId = $_.Group[0].SubscriptionId + $dupCount = $_.Count + Write-Error "Lab named $labName in subscription $subId is listed $dupCount times in Labs.json" + } + +# Iterate through all the labs +foreach ($selectedLab in $sortedLabList){ + + # Get the list of images in the current lab + SelectSubscription $selectedLab.SubscriptionId + $selectedLabRG = (Find-AzureRmResource -ResourceType 'Microsoft.DevTestLab/labs' | Where-Object { $_.Name -eq $selectedLab.LabName}).ResourceGroupName + + if($selectedLabRG) + { + Write-Output ("Found existing lab named $($selectedLab.LabName) in subscription $($selectedLab.SubscriptionId)") + } + else + { + Write-Error ("Unable to find an existing lab named $($selectedLab.LabName) in subscription $($selectedLab.SubscriptionId)") + } + + $goldenImagesFolder = Join-Path $ConfigurationLocation "GoldenImages" + $goldenImageFiles = Get-ChildItem $goldenImagesFolder -Recurse -Filter "*.json" | Select-Object FullName + foreach ($labImagePath in $selectedLab.ImagePaths){ + $filePath = Join-Path $goldenImagesFolder $labImagePath + $matchingImages = $goldenImageFiles | Where-Object {$_.FullName.StartsWith($filePath,"CurrentCultureIgnoreCase")} + if($matchingImages.Count -eq 0){ + Write-Error "The Lab named $($selectedLab.LabName) with SubscriptionId $($selectedLab.SubscriptionId) contains an ImagePath entry $labImagePath which does not point to any existing files in the GoldenImages folder." + } + } +}