diff --git a/build/pipelines/azure-pipelines.ci.yaml b/build/pipelines/azure-pipelines.ci.yaml index 19838c68..81adf58d 100644 --- a/build/pipelines/azure-pipelines.ci.yaml +++ b/build/pipelines/azure-pipelines.ci.yaml @@ -19,9 +19,42 @@ trigger: #- master jobs: -- job: Windows +- job: waitForRunningBuilds + displayName: 'Wait for Running Builds' pool: vmImage: 'windows-latest' + timeoutInMinutes: 360 + continueOnError: true + steps: + - template: ./templates/wait-runningBuild.yaml + parameters: + buildQueryPersonalAccessToken: $(BuildQueryPersonalAccessToken) + +- job: windowsWPS + displayName: 'Windows [Windows PowerShell]' + pool: + vmImage: 'windows-latest' + dependsOn: waitForRunningBuilds + steps: + - template: ./templates/verify-testConfigSettingsHash.yaml + parameters: + usePowerShellCore: false + - template: ./templates/run-staticAnalysis.yaml + parameters: + usePowerShellCore: false + - template: ./templates/run-unitTests.yaml + parameters: + gitHubAccessToken: $(WindowsPS5CIGitHubAccessToken) + gitHubOwnerName: $(WindowsPS5CIGitHubOwnerName) + gitHubOrganizationName: $(WindowsPS5CIGitHubOrganizationName) + platformName: 'Windows' + usePowerShellCore: false + +- job: windows + displayName: 'Windows [PowerShell Core]' + pool: + vmImage: 'windows-latest' + dependsOn: waitForRunningBuilds steps: - template: ./templates/verify-testConfigSettingsHash.yaml - template: ./templates/run-staticAnalysis.yaml @@ -32,9 +65,11 @@ jobs: gitHubOrganizationName: $(WindowsCIGitHubOrganizationName) platformName: 'Windows' -- job: Linux +- job: linux + displayName: 'Linux [PowerShell Core]' pool: vmImage: 'ubuntu-latest' + dependsOn: waitForRunningBuilds steps: - template: ./templates/verify-testConfigSettingsHash.yaml - template: ./templates/run-staticAnalysis.yaml @@ -46,8 +81,10 @@ jobs: platformName: 'Linux' - job: macOS + displayName: 'macOS [PowerShell Core]' pool: vmImage: 'macOS-latest' + dependsOn: waitForRunningBuilds steps: - template: ./templates/verify-testConfigSettingsHash.yaml - template: ./templates/run-staticAnalysis.yaml diff --git a/build/pipelines/templates/run-staticAnalysis.yaml b/build/pipelines/templates/run-staticAnalysis.yaml index 5325a767..9b9d6784 100644 --- a/build/pipelines/templates/run-staticAnalysis.yaml +++ b/build/pipelines/templates/run-staticAnalysis.yaml @@ -3,19 +3,34 @@ # report the results. # +parameters: +- name: 'usePowerShellCore' + default: true + type: boolean + steps: - - powershell: | - Install-Module -Name PSScriptAnalyzer -Repository PSGallery -Scope CurrentUser -Force -Verbose + - task: PowerShell@2 displayName: 'Install PSScriptAnalyzer' + inputs: + pwsh: eq('${{ parameters.usePowerShellCore }}', true) + errorActionPreference: 'stop' + targetType: 'inline' + script: | + Install-Module -Name PSScriptAnalyzer -Repository PSGallery -Scope CurrentUser -Force -Verbose - - powershell: | - $results = try { Invoke-ScriptAnalyzer -Settings ./PSScriptAnalyzerSettings.psd1 -Path ./ –Recurse -ErrorAction Stop } catch { 'Unexpected Error'; $_.Exception.StackTrace; if ($IsCoreCLR) { Get-Error }; throw } - $results | ForEach-Object { Write-Host "##vso[task.logissue type=$($_.Severity);sourcepath=$($_.ScriptPath);linenumber=$($_.Line);columnnumber=$($_.Column);]$($_.Message)" } - - $null = New-Item -Path ..\ -Name ScriptAnalyzer -ItemType Directory -Force - ./build/scripts/ConvertTo-NUnitXml.ps1 -ScriptAnalyzerResult $results -Path ../ScriptAnalyzer/test-results.xml - workingDirectory: '$(System.DefaultWorkingDirectory)' + - task: PowerShell@2 displayName: 'Run Static Code Analysis (PSScriptAnalyzer)' + inputs: + pwsh: eq('${{ parameters.usePowerShellCore }}', true) + errorActionPreference: 'stop' + workingDirectory: '$(System.DefaultWorkingDirectory)' + targetType: 'inline' + script: | + $results = try { Invoke-ScriptAnalyzer -Settings ./PSScriptAnalyzerSettings.psd1 -Path ./ –Recurse -ErrorAction Stop } catch { 'Unexpected Error'; $_.Exception.StackTrace; if ($IsCoreCLR) { Get-Error }; throw } + $results | ForEach-Object { Write-Host "##vso[task.logissue type=$($_.Severity);sourcepath=$($_.ScriptPath);linenumber=$($_.Line);columnnumber=$($_.Column);]$($_.Message)" } + + $null = New-Item -Path ..\ -Name ScriptAnalyzer -ItemType Directory -Force + ./build/scripts/ConvertTo-NUnitXml.ps1 -ScriptAnalyzerResult $results -Path ../ScriptAnalyzer/test-results.xml - task: PublishTestResults@2 displayName: 'Publish ScriptAnalyzer Test Results' diff --git a/build/pipelines/templates/run-unitTests.yaml b/build/pipelines/templates/run-unitTests.yaml index dc7d69ff..7b2b39fd 100644 --- a/build/pipelines/templates/run-unitTests.yaml +++ b/build/pipelines/templates/run-unitTests.yaml @@ -23,17 +23,31 @@ parameters: - name: 'platformName' default: 'Windows' type: string +- name: 'usePowerShellCore' + default: false + type: boolean steps: - - powershell: | - Install-Module -Name Pester -Repository PSGallery -Scope CurrentUser -AllowClobber -SkipPublisherCheck -RequiredVersion 4.10.1 -Force -Verbose + - task: PowerShell@2 displayName: 'Install Pester' + inputs: + pwsh: eq('${{ parameters.usePowerShellCore }}', true) + errorActionPreference: 'stop' + workingDirectory: '$(System.DefaultWorkingDirectory)' + targetType: 'inline' + script: | + Install-Module -Name Pester -Repository PSGallery -Scope CurrentUser -AllowClobber -SkipPublisherCheck -RequiredVersion 4.10.1 -Force -Verbose - - powershell: | - $null = New-Item -Path ..\ -Name Pester -ItemType Directory -Force - Invoke-Pester -CodeCoverage .\*.ps*1 -CodeCoverageOutputFile ../Pester/coverage.xml -CodeCoverageOutputFileFormat JaCoCo -EnableExit -Strict -OutputFile ../Pester/test-results.xml -OutputFormat NUnitXml - workingDirectory: '$(System.DefaultWorkingDirectory)' + - task: PowerShell@2 displayName: 'Run Unit Tests via Pester' + inputs: + pwsh: eq('${{ parameters.usePowerShellCore }}', true) + errorActionPreference: 'stop' + workingDirectory: '$(System.DefaultWorkingDirectory)' + targetType: 'inline' + script: | + $null = New-Item -Path ..\ -Name Pester -ItemType Directory -Force + Invoke-Pester -CodeCoverage .\*.ps*1 -CodeCoverageOutputFile ../Pester/coverage.xml -CodeCoverageOutputFileFormat JaCoCo -EnableExit -Strict -OutputFile ../Pester/test-results.xml -OutputFormat NUnitXml env: ciAccessToken: ${{ parameters.gitHubAccessToken }} ciOwnerName: ${{ parameters.gitHubOwnerName }} diff --git a/build/pipelines/templates/verify-testConfigSettingsHash.yaml b/build/pipelines/templates/verify-testConfigSettingsHash.yaml index 652aa144..a6065935 100644 --- a/build/pipelines/templates/verify-testConfigSettingsHash.yaml +++ b/build/pipelines/templates/verify-testConfigSettingsHash.yaml @@ -4,35 +4,45 @@ # the same in order to ensure the correct warning messages are presented to developers running UTs. # +parameters: +- name: 'usePowerShellCore' + default: true + type: boolean + steps: - - powershell: | - . ./Helpers.ps1 - $content = Get-Content -Path ./Tests/Config/Settings.ps1 -Raw -Encoding UTF8 - $hash = Get-SHA512Hash -PlainText $content - $configurationFile = Get-Content -Path ./GitHubConfiguration.ps1 -Raw -Encoding Utf8 - if($configurationFile -match "'testConfigSettingsHash' = '([^']+)'") - { - if($hash -ne $Matches[1]) + - task: PowerShell@2 + displayName: 'Set GitHubConfiguration.ps1 testConfigSettingsHash value.' + inputs: + pwsh: eq('${{ parameters.usePowerShellCore }}', true) + errorActionPreference: 'stop' + workingDirectory: '$(System.DefaultWorkingDirectory)' + targetType: 'inline' + script: | + . ./Helpers.ps1 + $content = Get-Content -Path ./Tests/Config/Settings.ps1 -Raw -Encoding UTF8 + $hash = Get-SHA512Hash -PlainText $content + $configurationFile = Get-Content -Path ./GitHubConfiguration.ps1 -Raw -Encoding Utf8 + if($configurationFile -match "'testConfigSettingsHash' = '([^']+)'") + { + if($hash -ne $Matches[1]) + { + $configHash = $Matches[1] + $message = @( + "`$testConfigSettingsHash value does not match the Get-SHA512Hash of file ./Tests/Config/Settings.ps1. If the contents of ", + "Settings.ps1 has been updated, please update the `$testConfigSettingsHash value in ./GitHubConfiguration.ps1's ", + "Import-GitHubConfiguration function. You can generate the new hash value using the commands:", + "", + "`t. ./Helpers.ps1", + "`tGet-SHA512Hash -PlainText (Get-Content -Path ./Tests/Config/Settings.ps1 -Raw -Encoding Utf8)", + "", + "`$testConfigSettingsHash = $configHash", + "Get-SHA512Hash(Settings.ps1) = $hash") + throw ($message -join [Environment]::NewLine) + } + } else { - $configHash = $Matches[1] $message = @( - "`$testConfigSettingsHash value does not match the Get-SHA512Hash of file ./Tests/Config/Settings.ps1. If the contents of ", - "Settings.ps1 has been updated, please update the `$testConfigSettingsHash value in ./GitHubConfiguration.ps1's ", - "Import-GitHubConfiguration function. You can generate the new hash value using the commands:", - "", - "`t. ./Helpers.ps1", - "`tGet-SHA512Hash -PlainText (Get-Content -Path ./Tests/Config/Settings.ps1 -Raw -Encoding Utf8)", - "", - "`$testConfigSettingsHash = $configHash", - "Get-SHA512Hash(Settings.ps1) = $hash") + "`$testConfigSettingsHash value not found in ./GitHubConfiguration.ps1. Please ensure ", + "the default hash value is set within the Import-GitHubConfiguration function.") throw ($message -join [Environment]::NewLine) } - } else - { - $message = @( - "`$testConfigSettingsHash value not found in ./GitHubConfiguration.ps1. Please ensure ", - "the default hash value is set within the Import-GitHubConfiguration function.") - throw ($message -join [Environment]::NewLine) - } - displayName: 'Set GitHubConfiguration.ps1 testConfigSettingsHash value.' - diff --git a/build/pipelines/templates/wait-runningBuild.yaml b/build/pipelines/templates/wait-runningBuild.yaml new file mode 100644 index 00000000..47306e17 --- /dev/null +++ b/build/pipelines/templates/wait-runningBuild.yaml @@ -0,0 +1,39 @@ +# +# This template contains the necessary jobs to create a queue for this pipeline and ensure that +# there is never more than one concurrent build actively running at any given time. +# + +#-------------------------------------------------------------------------------------------------- +# This template is dependent on the following pipeline variables being configured within the pipeline. +# +# 1. buildQueryPersonalAccessToken - An Azure DevOps Personal Access Token with Build READ permission. +# It should be configured as a "secret". +#-------------------------------------------------------------------------------------------------- + +parameters: +- name: 'buildQueryPersonalAccessToken' + type: string +- name: 'usePowerShellCore' + default: true + type: boolean + +steps: + - task: PowerShell@2 + displayName: 'Wait for previously queued builds' + inputs: + pwsh: eq('${{ parameters.usePowerShellCore }}', true) + errorActionPreference: 'stop' + workingDirectory: '$(System.DefaultWorkingDirectory)' + targetType: 'inline' + script: | + $params = @{ + PersonalAccessToken = $env:buildReadAccessToken + OrganizationName = 'ms' + ProjectName = 'PowerShellForGitHub' + BuildDefinitionId = $(System.DefinitionId) + BuildId = $(Build.BuildId) + } + + ./build/scripts/Wait-RunningBuild.ps1 @params + env: + buildReadAccessToken: ${{ parameters.buildQueryPersonalAccessToken }} diff --git a/build/scripts/Wait-RunningBuild.ps1 b/build/scripts/Wait-RunningBuild.ps1 new file mode 100644 index 00000000..9094506c --- /dev/null +++ b/build/scripts/Wait-RunningBuild.ps1 @@ -0,0 +1,168 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +<# + .DESCRIPTION + This script will enforce the concept of a queue for a build pipeline to ensure that + builds do not actively run concurrently. + + .PARAMETER PersonalAccessToken + A token that has Build READ permission. + + .PARAMETER OrganizationName + The name of the organization that this project is a part of. + + .PARAMETER ProjectName + The name of the project that this build pipeline can be found in. + + .PARAMETER BuildDefinitionId + The ID for the build definition that we are enforcing a queue on. + + .PARAMETER BuildId + The ID for this build. + + .PARAMETER NumSecondsSleepBetweenPolling + The number of seconds to sleep before polling attempt to check build pipeline status again. + + .PARAMETER MaxRetriesBeforeStarting + The number of successive retries that will be attempted to query for build pipeline status + before just allowing the build to start. + + .EXAMPLE + $params = @{ + PersonalAccessToken = $env:buildReadAccessToken + OrganizationName = 'ms' + ProjectName = 'PowerShellForGitHub' + BuildDefinitionId = $(System.DefinitionId) + BuildId = $(Build.BuildId) + } + ./Wait-RunningBuilds.ps1 @params +#> +[CmdletBinding()] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSAvoidUsingWriteHost", "", Justification = "This is the preferred way of writing output for Azure DevOps.")] +param( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] $PersonalAccessToken, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] $OrganizationName, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] $ProjectName, + + [Parameter(Mandatory)] + [int64] $BuildDefinitionId, + + [Parameter(Mandatory)] + [int64] $BuildId, + + [int] $NumSecondsSleepBetweenPolling = 30, + + [int] $MaxRetriesBeforeStarting = 3 +) + +Write-Host '[Wait-RunningBuilds] - Starting' + +$elapsedTimeFormat = '{0:hh\:mm\:ss}' +$stopwatch = New-Object -TypeName System.Diagnostics.Stopwatch +$stopwatch.Start() + +$url = "https://dev.azure.com/$OrganizationName/$ProjectName/_apis/build/builds?api-version=5.1&definitions=$BuildDefinitionId" +$headers = @{ + 'Authorization' = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":$PersonalAccessToken")) +} + +$remainingRetries = $MaxRetriesBeforeStarting +do +{ + try + { + $params = @{ + Uri = $url + Method = 'Get' + ContentType = 'application/json' + Headers = $headers + UseBasicParsing = $true + } + + $builds = Invoke-RestMethod @params + + $remainingRetries = $MaxRetriesBeforeStarting # successfully got a result. Reset remaining retries + + $thisBuild = $builds.value | Where-Object { $_.id -eq $BuildId } + $runningBuilds = @($builds.value | Where-Object { $_.status -eq 'inProgress' }) + $currentRunningBuild = $runningBuilds | Sort-Object -Property 'Id' | Select-Object -First 1 + + if ($null -eq $currentRunningBuild) + { + Write-Host 'Failed to identify the currently running build. To prevent an indefinite wait, allowing this build to start.' + break + } + elseif ($BuildId -ne $currentRunningBuild.id) + { + $buildsAheadInQueue = @($runningBuilds | Where-Object { $_.id -lt $BuildId }) + + # We want to display how long the current build has been running for _actively_, + # so we need to take into account if it had been queued while the previous build was + # running and thus subtract that extra time. + $currentRunningBuildStartTime = Get-Date -Date $currentRunningBuild.startTime + $lastCompletedBuild = $builds.value | + Where-Object { $_.status -ne 'inProgress' } | + Select-Object -First 1 + if ($null -ne $lastCompletedBuild) + { + $lastCompletedBuildTime = Get-Date -Date $lastCompletedBuild.finishTime + if ($lastCompletedBuildTime -gt $currentRunningBuildStartTime) + { + $waitedDuration = $lastCompletedBuildTime - $currentRunningBuildStartTime + $currentRunningBuildStartTime.AddMilliseconds($waitedDuration.TotalMilliseconds) + } + } + + $currentRunningBuildElapsedTime = New-TimeSpan -Start $currentRunningBuildStartTime -End (Get-Date) + $currentRunningBuildElapsedTimeFormatted = $elapsedTimeFormat -f $currentRunningBuildElapsedTime + + $timeWaited = New-TimeSpan -Start $thisBuild.startTime -End (Get-Date) + $timeWaitedFormatted = $elapsedTimeFormat -f $timeWaited + + $message = @( + "* Time: $(Get-Date -Format 'o')", + " This build: $($thisBuild.id) ($($thisBuild.buildNumber)) [Waiting for $timeWaitedFormatted]", + " Builds ahead in queue: $($buildsAheadInQueue.buildNumber -join ', ')", + " Total queued builds: $($runningBuilds.Count - 1)", + " Currently running build: $($currentRunningBuild.id) ($($currentRunningBuild.buildNumber)) [Running for $currentRunningBuildElapsedTimeFormatted]", + " Waiting $NumSecondsSleepBetweenPolling seconds before polling build status for this pipeline again...", + '--------------------------') + Write-Host ($message -join [Environment]::NewLine) + } + else + { + break + } + } + catch + { + $remainingRetries-- + if ($remainingRetries -lt 0) + { + Write-Host 'Still unable to retrieve build status for this pipeline. Exhausted retries. To prevent an indefinite wait, allowing this build to start.' + break + } + else + { + Write-Host "Failed to get build status for this pipeline. Will try again in $NumSecondsSleepBetweenPolling seconds. $remainingRetries retries remaining." + } + } + + Start-Sleep -Seconds $NumSecondsSleepBetweenPolling +} +while ($true) + +$stopwatch.Stop() +$timeWaitedFormatted = $elapsedTimeFormat -f $stopwatch.Elapsed +Write-Host "Waiting completed after $timeWaitedFormatted. Starting this build." + +Write-Host '[Wait-RunningBuilds] - Exiting' \ No newline at end of file