Update pwsh to 7.1.14, separate image minting from vmss minting, and …

scripts/azure-pipelines/azure-pipelines.yml

@@ -2,8 +2,8 @@
 # SPDX-License-Identifier: MIT
 #
 variables:
-  windows-pool: 'PrWin-2021-08-12'
-  linux-pool: 'PrLin-2021-08-11'
+  windows-pool: 'PrWin-2021-09-08'
+  linux-pool: 'PrLin-2021-09-08'
   osx-pool: 'PrOsx-2021-07-27'
 
 stages:

scripts/azure-pipelines/create-vmss-helpers.psm1

@@ -55,6 +55,60 @@ function Find-ResourceGroupName {
   return $result
 }
 
+<#
+.SYNOPSIS
+Returns whether there's a name collision for an image in the resource group.
+
+.DESCRIPTION
+Find-ImageNameCollision takes a list of images, and checks if $Test
+collides names with any of the image names.
+
+.PARAMETER Test
+The name to test.
+
+.PARAMETER Images
+The list of images.
+#>
+function Find-ImageNameCollision {
+  [CmdletBinding()]
+  Param([string]$Test, $Images)
+
+  foreach ($resource in $Images) {
+    if ($resource.Name -eq $Test) {
+      return $true
+    }
+  }
+
+  return $false
+}
+
+<#
+.SYNOPSIS
+Attempts to find a name that does not collide with any images in the resource group.
+
+.DESCRIPTION
+Find-ResourceGroupName takes a set of resources from Get-AzResourceGroup, and finds the
+first name in {$Prefix, $Prefix-1, $Prefix-2, ...} such that the name doesn't collide with
+any of the resources in the resource group.
+
+.PARAMETER Prefix
+The prefix of the final name; the returned name will be of the form "$Prefix(-[1-9][0-9]*)?"
+#>
+function Find-ImageName {
+  [CmdLetBinding()]
+  Param([string]$ResourceGroupName, [string]$Prefix)
+
+  $images = Get-AzImage -ResourceGroupName $ResourceGroupName
+  $result = $Prefix
+  $suffix = 0
+  while (Find-ImageNameCollision -Test $result -Images $images) {
+    $suffix++
+    $result = "$Prefix-$suffix"
+  }
+
+  return $result
+}
+
 <#
 .SYNOPSIS
 Generates a random password.
@@ -160,7 +214,135 @@ function Sanitize-Name {
   return $result
 }
 
+<#
+.SYNOPSIS
+Creates a new Azure virtual network with locked down firewall rules.
+
+.PARAMETER ResourceGroupName
+The name of the resource group in which the virtual network should be created.
+
+.PARAMETER Location
+The location (region) where the network is to be created.
+#>
+function Create-LockedDownNetwork {
+  [CmdletBinding()]
+  Param(
+    [parameter(Mandatory=$true)]
+    [string]$ResourceGroupName,
+    [parameter(Mandatory=$true)]
+    [string]$Location
+  )
+
+  $allFirewallRules = @()
+
+  $allFirewallRules += New-AzNetworkSecurityRuleConfig `
+    -Name AllowHTTP `
+    -Description 'Allow HTTP(S)' `
+    -Access Allow `
+    -Protocol Tcp `
+    -Direction Outbound `
+    -Priority 1008 `
+    -SourceAddressPrefix * `
+    -SourcePortRange * `
+    -DestinationAddressPrefix * `
+    -DestinationPortRange @(80, 443)
+
+  $allFirewallRules += New-AzNetworkSecurityRuleConfig `
+    -Name AllowSFTP `
+    -Description 'Allow (S)FTP' `
+    -Access Allow `
+    -Protocol Tcp `
+    -Direction Outbound `
+    -Priority 1009 `
+    -SourceAddressPrefix * `
+    -SourcePortRange * `
+    -DestinationAddressPrefix * `
+    -DestinationPortRange @(21, 22)
+
+  $allFirewallRules += New-AzNetworkSecurityRuleConfig `
+    -Name AllowDNS `
+    -Description 'Allow DNS' `
+    -Access Allow `
+    -Protocol * `
+    -Direction Outbound `
+    -Priority 1010 `
+    -SourceAddressPrefix * `
+    -SourcePortRange * `
+    -DestinationAddressPrefix * `
+    -DestinationPortRange 53
+
+  $allFirewallRules += New-AzNetworkSecurityRuleConfig `
+    -Name AllowGit `
+    -Description 'Allow git' `
+    -Access Allow `
+    -Protocol Tcp `
+    -Direction Outbound `
+    -Priority 1011 `
+    -SourceAddressPrefix * `
+    -SourcePortRange * `
+    -DestinationAddressPrefix * `
+    -DestinationPortRange 9418
+
+  $allFirewallRules += New-AzNetworkSecurityRuleConfig `
+    -Name DenyElse `
+    -Description 'Deny everything else' `
+    -Access Deny `
+    -Protocol * `
+    -Direction Outbound `
+    -Priority 1013 `
+    -SourceAddressPrefix * `
+    -SourcePortRange * `
+    -DestinationAddressPrefix * `
+    -DestinationPortRange *
+
+  $NetworkSecurityGroupName = $ResourceGroupName + 'NetworkSecurity'
+  $NetworkSecurityGroup = New-AzNetworkSecurityGroup `
+    -Name $NetworkSecurityGroupName `
+    -ResourceGroupName $ResourceGroupName `
+    -Location $Location `
+    -SecurityRules $allFirewallRules
+
+  $SubnetName = $ResourceGroupName + 'Subnet'
+  $Subnet = New-AzVirtualNetworkSubnetConfig `
+    -Name $SubnetName `
+    -AddressPrefix "10.0.0.0/16" `
+    -NetworkSecurityGroup $NetworkSecurityGroup `
+    -ServiceEndpoint "Microsoft.Storage"
+
+  $VirtualNetworkName = $ResourceGroupName + 'Network'
+  $VirtualNetwork = New-AzVirtualNetwork `
+    -Name $VirtualNetworkName `
+    -ResourceGroupName $ResourceGroupName `
+    -Location $Location `
+    -AddressPrefix "10.0.0.0/16" `
+    -Subnet $Subnet
+
+  return $VirtualNetwork
+}
+
+function Invoke-AzVMRunCommandWithRetries {
+  try {
+    return Invoke-AzVMRunCommand @args
+  } catch {
+    for ($idx = 0; $idx -lt 5; $idx++) {
+      Write-Host "Running command failed. $_ Retrying after 10 seconds..."
+      Start-Sleep -Seconds 10
+      try {
+        return Invoke-AzVMRunCommand @args
+      } catch {
+        # ignore
+      }
+    }
+
+    Write-Host "Running command failed too many times. Giving up!"
+    throw $_
+  }
+}
+
 Export-ModuleMember -Function Find-ResourceGroupName
+Export-ModuleMember -Function Find-ImageName
 Export-ModuleMember -Function New-Password
 Export-ModuleMember -Function Wait-Shutdown
 Export-ModuleMember -Function Sanitize-Name
+Export-ModuleMember -Function Create-LockedDownNetwork
+Export-ModuleMember -Function Invoke-AzVMRunCommandWithRetries

scripts/azure-pipelines/linux/create-image.ps1

@@ -0,0 +1,161 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+<#
+.SYNOPSIS
+Creates a Linux virtual machine image, set up for vcpkg's CI.
+
+.DESCRIPTION
+create-image.ps1 creates an Azure Linux VM image, set up for vcpkg's CI system.
+This script assumes you have installed Azure tools into PowerShell by following the instructions
+at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1
+or are running from Azure Cloud Shell.
+
+This script assumes you have installed the OpenSSH Client optional Windows component.
+#>
+
+$Location = 'westus2'
+$Prefix = 'Lin-'
+$Prefix += (Get-Date -Format 'yyyy-MM-dd')
+$VMSize = 'Standard_D32as_v4'
+$ProtoVMName = 'PROTOTYPE'
+$ErrorActionPreference = 'Stop'
+
+$ProgressActivity = 'Creating Linux Image'
+$TotalProgress = 9
+$CurrentProgress = 1
+
+Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Creating SSH key' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$sshDir = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName()
+mkdir $sshDir
+try {
+  ssh-keygen.exe -q -b 2048 -t rsa -f "$sshDir/key" -P [string]::Empty
+  $sshPublicKey = Get-Content "$sshDir/key.pub"
+} finally {
+  Remove-Item $sshDir -Recurse -Force
+}
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Creating resource group' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$ResourceGroupName = Find-ResourceGroupName $Prefix
+$AdminPW = New-Password
+New-AzResourceGroup -Name $ResourceGroupName -Location $Location
+$AdminPWSecure = ConvertTo-SecureString $AdminPW -AsPlainText -Force
+$Credential = New-Object System.Management.Automation.PSCredential ("AdminUser", $AdminPWSecure)
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Creating virtual network' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Creating prototype VM' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$NicName = $ResourceGroupName + 'NIC'
+$Nic = New-AzNetworkInterface `
+  -Name $NicName `
+  -ResourceGroupName $ResourceGroupName `
+  -Location $Location `
+  -Subnet $VirtualNetwork.Subnets[0]
+
+$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1
+$VM = Set-AzVMOperatingSystem `
+  -VM $VM `
+  -Linux `
+  -ComputerName $ProtoVMName `
+  -Credential $Credential `
+  -DisablePasswordAuthentication
+
+$VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id
+$VM = Set-AzVMSourceImage `
+  -VM $VM `
+  -PublisherName 'Canonical' `
+  -Offer '0001-com-ubuntu-server-focal' `
+  -Skus '20_04-lts-gen2' `
+  -Version latest
+
+$VM = Set-AzVMBootDiagnostic -VM $VM -Disable
+
+$VM = Add-AzVMSshPublicKey `
+  -VM $VM `
+  -KeyData $sshPublicKey `
+  -Path "/home/AdminUser/.ssh/authorized_keys"
+
+New-AzVm `
+  -ResourceGroupName $ResourceGroupName `
+  -Location $Location `
+  -VM $VM
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Running provisioning script provision-image.sh in VM' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+$ProvisionImageResult = Invoke-AzVMRunCommandWithRetries `
+  -ResourceGroupName $ResourceGroupName `
+  -VMName $ProtoVMName `
+  -CommandId 'RunShellScript' `
+  -ScriptPath "$PSScriptRoot\provision-image.sh"
+
+Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Restarting VM' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Converting VM to Image' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Stop-AzVM `
+  -ResourceGroupName $ResourceGroupName `
+  -Name $ProtoVMName `
+  -Force
+
+Set-AzVM `
+  -ResourceGroupName $ResourceGroupName `
+  -Name $ProtoVMName `
+  -Generalized
+
+$VM = Get-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
+$ImageConfig = New-AzImageConfig -Location $Location -SourceVirtualMachineId $VM.ID -HyperVGeneration 'V2'
+$ImageName = Find-ImageName -ResourceGroupName 'vcpkg-image-minting' -Prefix $Prefix
+New-AzImage -Image $ImageConfig -ImageName $ImageName -ResourceGroupName 'vcpkg-image-minting'
+
+####################################################################################################
+Write-Progress `
+  -Activity $ProgressActivity `
+  -Status 'Deleting unused temporary resources' `
+  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+
+Remove-AzResourceGroup $ResourceGroupName -Force
+
+####################################################################################################
+Write-Progress -Activity $ProgressActivity -Completed
+Write-Host "Generated Image:  $ImageName"
+Write-Host 'Finished!'