diff --git a/scripts/azure-pipelines/azure-pipelines-osx.yml b/scripts/azure-pipelines/azure-pipelines-osx.yml new file mode 100644 index 00000000000000..d2aa2ad9204b26 --- /dev/null +++ b/scripts/azure-pipelines/azure-pipelines-osx.yml @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# +variables: + osx-pool: 'PrOsx-2021-07-27' + +jobs: +- template: osx/azure-pipelines.yml + parameters: + poolName: $(osx-pool) diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index a4ac11d5679e60..561d19fe90c44b 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -49,19 +49,10 @@ jobs: jobName: arm_uwp poolName: $(windows-pool) -- template: osx/azure-pipelines.yml - parameters: - poolName: $(osx-pool) - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - mode: AllPorts - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - mode: NonTensorflow -- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: +- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - template: osx/azure-pipelines.yml parameters: poolName: $(osx-pool) - mode: OnlyTensorflow - jobName: x64_osx_tensorflow - template: linux/azure-pipelines.yml parameters: diff --git a/scripts/azure-pipelines/generate-skip-list.ps1 b/scripts/azure-pipelines/generate-skip-list.ps1 index 6c533595faf9c9..84b78b33878721 100755 --- a/scripts/azure-pipelines/generate-skip-list.ps1 +++ b/scripts/azure-pipelines/generate-skip-list.ps1 @@ -20,8 +20,7 @@ The path to the ci.baseline.txt file. Param( [string]$Triplet, [string]$BaselineFile, - [switch]$SkipFailures = $false, - [String[]]$AdditionalSkips = @() + [switch]$SkipFailures = $false ) $ErrorActionPreference = 'Stop' @@ -81,5 +80,4 @@ if ($SkipFailures) { $skip_list = $baselineForTriplet ` | Where-Object { $_ -match $targetRegex } ` | ForEach-Object { $_ -replace ":.*$" } -$skip_list += $AdditionalSkips [string]::Join(",", $skip_list) diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index a8d94f6f647b3e..8749826188ce15 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -5,23 +5,14 @@ parameters: - name: poolName type: string -- name: mode - type: string - values: - - 'AllPorts' - - 'NonTensorflow' - - 'OnlyTensorflow' -- name: jobName - type: string - default: x64_osx jobs: -- job: "${{ parameters.jobName }}" +- job: x64_osx pool: name: ${{ parameters.poolName }} workspace: clean: resources - timeoutInMinutes: 1440 # 1 day + timeoutInMinutes: 2880 # 2 days variables: - name: WORKING_ROOT value: /Users/vagrant/Data @@ -33,18 +24,6 @@ jobs: - group: vcpkg-asset-caching-credentials - name: X_VCPKG_ASSET_SOURCES value: "x-azurl,$(root-url),$(sas),readwrite" - - name: PowershellExtraArguments - ${{ if eq(parameters.mode, 'AllPorts') }}: - value: '' - ${{ if eq(parameters.mode, 'NonTensorflow') }}: - value: "-AdditionalSkips @('tensorflow', 'tensorflow-cc')" - ${{ if eq(parameters.mode, 'OnlyTensorflow') }}: - value: "-OnlyTest @('tensorflow', 'tensorflow-cc')" - - name: Postfix - ${{ if eq(parameters.mode, 'OnlyTensorflow') }}: - value: '-tensorflow' - ${{ if ne(parameters.mode, 'OnlyTensorflow') }}: - value: '' steps: - bash: df -h @@ -70,7 +49,6 @@ jobs: -BinarySourceStub "${{ variables.X_VCPKG_BINARY_SOURCE_STUB }}" -WorkingRoot "${{ variables.WORKING_ROOT }}" -ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)" - ${{ variables.PowershellExtraArguments }} pwsh: true - bash: | df -h diff --git a/scripts/azure-pipelines/test-modified-ports.ps1 b/scripts/azure-pipelines/test-modified-ports.ps1 index c17010cab5569a..5126c9080b31f4 100755 --- a/scripts/azure-pipelines/test-modified-ports.ps1 +++ b/scripts/azure-pipelines/test-modified-ports.ps1 @@ -52,8 +52,6 @@ Param( [Parameter(ParameterSetName='BinarySourceStub')] $BinarySourceStub = $null, [String]$BuildReason = $null, - [String[]]$AdditionalSkips = @(), - [String[]]$OnlyTest = $null, [switch]$NoParentHashes = $false, [switch]$PassingIsPassing = $false ) @@ -141,66 +139,46 @@ if ($LASTEXITCODE -ne 0) $skipList = . "$PSScriptRoot/generate-skip-list.ps1" ` -Triplet $Triplet ` -BaselineFile "$PSScriptRoot/../ci.baseline.txt" ` - -SkipFailures:$skipFailures ` - -AdditionalSkips $AdditionalSkips + -SkipFailures:$skipFailures -if ($null -ne $OnlyTest) +$hostArgs = @() +if ($Triplet -in @('x64-windows', 'x64-osx', 'x64-linux')) { - $OnlyTest | % { - $portName = $_ - & "./vcpkg$executableExtension" install --triplet $Triplet @commonArgs @cachingArgs $portName - if (-not $?) - { - [System.Console]::Error.WriteLine( ` - "REGRESSION: ${portName}:$triplet. If expected, remove ${portName} from the OnlyTest list." ` - ) - } - } - - $failureLogsEmpty = (-Not (Test-Path $failureLogs) -Or ((Get-ChildItem $failureLogs).count -eq 0)) - Write-Host "##vso[task.setvariable variable=FAILURE_LOGS_EMPTY]$failureLogsEmpty" + # WORKAROUND: These triplets are native-targetting which triggers an issue in how vcpkg handles the skip list. + # The workaround is to pass the skip list as host-excludes as well. + $hostArgs = @("--host-exclude=$skipList") } -else -{ - $hostArgs = @() - if ($Triplet -in @('x64-windows', 'x64-osx', 'x64-linux')) - { - # WORKAROUND: These triplets are native-targetting which triggers an issue in how vcpkg handles the skip list. - # The workaround is to pass the skip list as host-excludes as well. - $hostArgs = @("--host-exclude=$skipList") - } - - $parentHashes = @() - if (($BuildReason -eq 'PullRequest') -and -not $NoParentHashes) - { - # Prefetch tools for better output - & "./vcpkg$executableExtension" fetch cmake - & "./vcpkg$executableExtension" fetch ninja - & "./vcpkg$executableExtension" fetch git - - Write-Host "Determining parent hashes using HEAD~1" - $parentHashesFile = Join-Path $WorkingRoot 'parent-hashes.json' - $parentHashes = @("--parent-hashes=$parentHashesFile") - & git revert -n -m 1 HEAD - & "./vcpkg$executableExtension" ci $Triplet --dry-run --exclude=$skipList @hostArgs @commonArgs --no-binarycaching "--output-hashes=$parentHashesFile" ` - | ForEach-Object { if ($_ -match ' dependency information| determine pass') { Write-Host $_ } } - & git reset --hard HEAD - - Write-Host "Running CI using parent hashes" - } - & "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList --failure-logs=$failureLogs @hostArgs @commonArgs @cachingArgs @parentHashes +$parentHashes = @() +if (($BuildReason -eq 'PullRequest') -and -not $NoParentHashes) +{ + # Prefetch tools for better output + & "./vcpkg$executableExtension" fetch cmake + & "./vcpkg$executableExtension" fetch ninja + & "./vcpkg$executableExtension" fetch git + + Write-Host "Determining parent hashes using HEAD~1" + $parentHashesFile = Join-Path $WorkingRoot 'parent-hashes.json' + $parentHashes = @("--parent-hashes=$parentHashesFile") + & git revert -n -m 1 HEAD + & "./vcpkg$executableExtension" ci $Triplet --dry-run --exclude=$skipList @hostArgs @commonArgs --no-binarycaching "--output-hashes=$parentHashesFile" ` + | ForEach-Object { if ($_ -match ' dependency information| determine pass') { Write-Host $_ } } + & git reset --hard HEAD + + Write-Host "Running CI using parent hashes" +} - $failureLogsEmpty = (-Not (Test-Path $failureLogs) -Or ((Get-ChildItem $failureLogs).count -eq 0)) - Write-Host "##vso[task.setvariable variable=FAILURE_LOGS_EMPTY]$failureLogsEmpty" +& "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList --failure-logs=$failureLogs @hostArgs @commonArgs @cachingArgs @parentHashes - if ($LASTEXITCODE -ne 0) - { - throw "vcpkg ci failed" - } +$failureLogsEmpty = (-Not (Test-Path $failureLogs) -Or ((Get-ChildItem $failureLogs).count -eq 0)) +Write-Host "##vso[task.setvariable variable=FAILURE_LOGS_EMPTY]$failureLogsEmpty" - & "$PSScriptRoot/analyze-test-results.ps1" -logDir $xmlResults ` - -triplet $Triplet ` - -baselineFile .\scripts\ci.baseline.txt ` - -passingIsPassing:$PassingIsPassing +if ($LASTEXITCODE -ne 0) +{ + throw "vcpkg ci failed" } + +& "$PSScriptRoot/analyze-test-results.ps1" -logDir $xmlResults ` + -triplet $Triplet ` + -baselineFile .\scripts\ci.baseline.txt ` + -passingIsPassing:$PassingIsPassing