From f0f17b4981e70505e9bf44b9c30841ba62fabd8c Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 2 Nov 2020 12:50:51 -0700 Subject: [PATCH 01/99] Use scale set agent pool for official builds --- azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 5515aa48..49bbf65b 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -38,7 +38,7 @@ stages: jobs: - template: build.yml parameters: - windowsPool: VSEng-MicroBuildVS2019 + windowsPool: VSEngSS-MicroBuild2019 - stage: symbol_archive displayName: Symbol archival From 14dc279ad8ce1a6d85706ddd8aadf7bed47decbe Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 1 Dec 2020 14:53:50 -0700 Subject: [PATCH 02/99] Add stock MSFT starter files --- CODE_OF_CONDUCT.md | 9 +++++++++ SECURITY.md | 41 +++++++++++++++++++++++++++++++++++++++++ SUPPORT.md | 25 +++++++++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 CODE_OF_CONDUCT.md create mode 100644 SECURITY.md create mode 100644 SUPPORT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..f9ba8cf6 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,9 @@ +# Microsoft Open Source Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..f7b89984 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). + + \ No newline at end of file diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 00000000..8b05616f --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,25 @@ +# TODO: The maintainer of this repo has not yet edited this file + +**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? + +- **No CSS support:** Fill out this template with information about how to file issues and get help. +- **Yes CSS support:** Fill out an intake form at [aka.ms/spot](https://aka.ms/spot). CSS will work with/help you to determine next steps. More details also available at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). +- **Not sure?** Fill out a SPOT intake as though the answer were "Yes". CSS will help you decide. + +*Then remove this first heading from this SUPPORT.MD file before publishing your repo.* + +# Support + +## How to file issues and get help + +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. + +For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE +FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER +CHANNEL. WHERE WILL YOU HELP PEOPLE?**. + +## Microsoft Support Policy + +Support for this **PROJECT or PRODUCT** is limited to the resources listed above. From eda964053b115e4adc55a34c70058b8d8e68eff4 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 9 Dec 2020 12:52:35 -0700 Subject: [PATCH 03/99] Update MicroBuild Insert task version --- azure-pipelines/vs-insertion.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index 75c54f31..90645cb2 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -37,7 +37,7 @@ stages: packagesToPush: $(Pipeline.Workspace)/CI/VSInsertion-windows/*.nupkg publishVstsFeed: 97a41293-2972-4f48-8c0e-05493ae82010 allowPackageConflicts: true - - task: MicroBuildInsertVsPayload@3 + - task: MicroBuildInsertVsPayload@4 displayName: Insert VS Payload inputs: TeamName: $(TeamName) From ee54c45f8f038d108bfbd4a1273b0f5d80553cf9 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 9 Dec 2020 13:08:36 -0700 Subject: [PATCH 04/99] Remove code to push to CI The feeds it was pushing to were obsolete anyway. --- azure-pipelines/official.yml | 1 - azure-pipelines/publish-deployables.yml | 10 ---------- azure-pipelines/variables/ci_feed.ps1 | 8 -------- 3 files changed, 19 deletions(-) delete mode 100644 azure-pipelines/variables/ci_feed.ps1 diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 5515aa48..e19311a6 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -31,7 +31,6 @@ stages: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release BuildPlatform: Any CPU - push_to_ci: true NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages SignTypeSelection: ${{ parameters.SignTypeSelection }} diff --git a/azure-pipelines/publish-deployables.yml b/azure-pipelines/publish-deployables.yml index 9cb41fc6..5a981b26 100644 --- a/azure-pipelines/publish-deployables.yml +++ b/azure-pipelines/publish-deployables.yml @@ -7,13 +7,3 @@ steps: displayName: Use NuGet 5.x inputs: versionSpec: 5.x - -- task: NuGetCommand@2 - displayName: Push packages to CI feed - inputs: - command: push - packagesToPush: $(Pipeline.Workspace)/deployables-Windows/NuGet/*.nupkg - nuGetFeedType: internal - publishVstsFeed: $(ci_feed) - allowPackageConflicts: true - condition: and(succeeded(), eq(variables['push_to_ci'], 'true'), ne(variables['ci_feed'], ''), ne(variables['Build.Reason'], 'PullRequest')) diff --git a/azure-pipelines/variables/ci_feed.ps1 b/azure-pipelines/variables/ci_feed.ps1 deleted file mode 100644 index 82bad79d..00000000 --- a/azure-pipelines/variables/ci_feed.ps1 +++ /dev/null @@ -1,8 +0,0 @@ -$signType = $env:SIGNTYPE -if (!$signType) { $signType = & "$PSScriptRoot/SignType.ps1" } - -if ($signType -eq 'Real') { - '09d8d03c-1ac8-456e-9274-4d2364527d99' # VSIDE-RealSigned-Release -} else { - 'da484c78-f942-44ef-b197-99e2a1bef53c' # VSIDE-TestSigned-Release -} From 9ecac8356a944264348485e22e2fbfd9ec41cbe9 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 9 Dec 2020 13:13:46 -0700 Subject: [PATCH 05/99] Update team name --- azure-pipelines/variables/TeamName.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/variables/TeamName.ps1 b/azure-pipelines/variables/TeamName.ps1 index 871749a5..d3e73642 100644 --- a/azure-pipelines/variables/TeamName.ps1 +++ b/azure-pipelines/variables/TeamName.ps1 @@ -1 +1 @@ -'VS IDE' +'VS Core' From a6ae0b843a96eeed123234e9e6e6f6f6dce469c6 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 9 Dec 2020 20:25:40 -0700 Subject: [PATCH 06/99] Bring back code to support CI feeds, but just remove our existing CI feed. --- azure-pipelines/official.yml | 1 + azure-pipelines/publish-deployables.yml | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index e19311a6..5515aa48 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -31,6 +31,7 @@ stages: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release BuildPlatform: Any CPU + push_to_ci: true NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages SignTypeSelection: ${{ parameters.SignTypeSelection }} diff --git a/azure-pipelines/publish-deployables.yml b/azure-pipelines/publish-deployables.yml index 5a981b26..9cb41fc6 100644 --- a/azure-pipelines/publish-deployables.yml +++ b/azure-pipelines/publish-deployables.yml @@ -7,3 +7,13 @@ steps: displayName: Use NuGet 5.x inputs: versionSpec: 5.x + +- task: NuGetCommand@2 + displayName: Push packages to CI feed + inputs: + command: push + packagesToPush: $(Pipeline.Workspace)/deployables-Windows/NuGet/*.nupkg + nuGetFeedType: internal + publishVstsFeed: $(ci_feed) + allowPackageConflicts: true + condition: and(succeeded(), eq(variables['push_to_ci'], 'true'), ne(variables['ci_feed'], ''), ne(variables['Build.Reason'], 'PullRequest')) From 3fa4f21b2326fb79aa47ceb8c7b5e2743ff7fb7d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 9 Dec 2020 21:04:18 -0700 Subject: [PATCH 07/99] Add missing includeMacOS switch to official builds --- azure-pipelines/official.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 5515aa48..82532aa5 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -22,6 +22,10 @@ parameters: type: string default: Test values: [ 'Test', 'Real' ] +- name: includeMacOS + displayName: Build on macOS + type: boolean + default: false # macOS is often bogged down in Azure Pipelines stages: @@ -39,6 +43,7 @@ stages: - template: build.yml parameters: windowsPool: VSEng-MicroBuildVS2019 + includeMacOS: ${{ parameters.includeMacOS }} - stage: symbol_archive displayName: Symbol archival From bb90bbb9fbe6b6c69e7ea5392a7d334bf6c8286a Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 9 Dec 2020 21:24:59 -0700 Subject: [PATCH 08/99] Revert team name change It breaks symbol archival --- azure-pipelines/variables/TeamName.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines/variables/TeamName.ps1 b/azure-pipelines/variables/TeamName.ps1 index d3e73642..5f2822c5 100644 --- a/azure-pipelines/variables/TeamName.ps1 +++ b/azure-pipelines/variables/TeamName.ps1 @@ -1 +1,2 @@ -'VS Core' +# This value is used to craft a \\cpvsbuild\drops path for symbol archival. +'VS IDE' From f29842186805f5522ff168e2b4c61337c42759fd Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 18 Dec 2020 17:52:49 -0700 Subject: [PATCH 09/99] Fix display name of symbol archival task --- azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 82532aa5..692d3deb 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -64,7 +64,7 @@ stages: artifact: symbols-Windows displayName: Download symbols-Windows artifact - task: MicroBuildArchiveSymbols@1 - displayName: Archive $(SymbolsFeatureName) on Symweb + displayName: Archive symbols to Symweb inputs: SymbolsFeatureName: $(SymbolsFeatureName) SymbolsSymwebProject: VS From 3639f0321c4fb5bf4153f6c84c3e91cff68b244c Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 15 Jan 2021 10:32:24 -0700 Subject: [PATCH 10/99] Bump version of codesign verify task --- azure-pipelines/microbuild.after.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 6e29912b..55fee5ec 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -1,5 +1,5 @@ steps: -- task: MicroBuildCodesignVerify@2 +- task: MicroBuildCodesignVerify@3 displayName: Verify Signed Files inputs: TargetFolders: | From 15802d7d14fb2b3865b27e4099ce2fd05fdf9bb4 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 15 Jan 2021 12:19:50 -0700 Subject: [PATCH 11/99] Clean up trailing line endings --- SUPPORT.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/SUPPORT.md b/SUPPORT.md index 8b05616f..ecbe64c0 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -10,16 +10,16 @@ # Support -## How to file issues and get help +## How to file issues and get help -This project uses GitHub Issues to track bugs and feature requests. Please search the existing -issues before filing new issues to avoid duplicates. For new issues, file your bug or +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or feature request as a new Issue. -For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE +For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER CHANNEL. WHERE WILL YOU HELP PEOPLE?**. -## Microsoft Support Policy +## Microsoft Support Policy Support for this **PROJECT or PRODUCT** is limited to the resources listed above. From f52b382221eba041c184e3a70268d7799f52841d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 15 Jan 2021 12:19:50 -0700 Subject: [PATCH 12/99] Clean up trailing line endings --- azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 737df08a..f2336144 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -68,7 +68,7 @@ stages: inputs: SymbolsFeatureName: $(SymbolsFeatureName) SymbolsSymwebProject: VS - SymbolsUncPath: \\cpvsbuild\drops\$(TeamName)\$(Build.DefinitionName)\$(Build.SourceBranchName)\$(Build.BuildNumber)\Symbols.Archival + SymbolsUncPath: \\cpvsbuild\drops\$(TeamName)\$(Build.DefinitionName)\$(Build.SourceBranchName)\$(Build.BuildId)\Symbols.Archival SymbolsEmailContacts: vsidemicrobuild SymbolsAgentPath: $(Pipeline.Workspace)/symbols-Windows - task: MicroBuildCleanup@1 From ecc46dc8f71c838d66ca1948fd70336d1c2f2e49 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 1 Feb 2021 16:13:29 -0700 Subject: [PATCH 13/99] Consume new Microbuild package IDs --- Directory.Build.props | 4 ++-- test/Directory.Build.props | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index ce25ce63..4d0eff73 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -19,11 +19,11 @@ true snupkg - 2.0.61 + 2.0.65 - + diff --git a/test/Directory.Build.props b/test/Directory.Build.props index 8edee731..0f7ca578 100644 --- a/test/Directory.Build.props +++ b/test/Directory.Build.props @@ -8,6 +8,6 @@ - + From e4eaa925a2327ec9d23a17fb08cec013121b1352 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 5 Feb 2021 17:33:22 -0700 Subject: [PATCH 14/99] Adjust for main as default branch --- azure-pipelines/microbuild.after.yml | 2 +- azure-pipelines/official.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 55fee5ec..853b6117 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -23,7 +23,7 @@ steps: /repoName:$(Build.Repository.Name) /additionalCodexArguments:-bld /additionalCodexArguments:$(Build.ArtifactStagingDirectory)/build_logs - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), ne(variables['Build.Reason'], 'PullRequest')) + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'), ne(variables['Build.Reason'], 'PullRequest')) continueOnError: true - template: secure-development-tools.yml diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index f2336144..b1cddc15 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -1,7 +1,7 @@ trigger: branches: include: - - master + - main - microbuild - 'validate/*' paths: From 0cfb17c6eb4955b5c96830af3ee364b24b8944e8 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 4 Feb 2021 17:41:12 -0700 Subject: [PATCH 15/99] Secure nuget packages to require signatures by known parties --- azure-pipelines/official.yml | 3 +++ nuget.config | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index b1cddc15..d00f13df 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -27,6 +27,9 @@ parameters: type: boolean default: false # macOS is often bogged down in Azure Pipelines +variables: + NugetSecurityAnalysisWarningLevel: none # nuget.config requires signed packages by trusted owners + stages: - stage: Build diff --git a/nuget.config b/nuget.config index 57d03218..90f832a9 100644 --- a/nuget.config +++ b/nuget.config @@ -1,7 +1,8 @@ - +๏ปฟ + @@ -13,4 +14,13 @@ + + + Microsoft;xunit;manuel.roemer;sharwell;jamesnk;aarnott;MarcoRossignoli;Thecentury + + + + + + From af7742d884fdbed4e517da5bb9c62365746f5d19 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 24 Feb 2021 09:51:59 -0700 Subject: [PATCH 16/99] Use `dotnet build` for official builds --- azure-pipelines/dotnet.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index 51d35eed..958154a0 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -1,20 +1,9 @@ steps: -# We use VSBuild instead of "dotnet build" on Windows where MicroBuild tasks have to run (since they don't support MSBuild Core yet). -- task: VSBuild@1 - displayName: Build Visual Studio solution - inputs: - msbuildArgs: /t:build,pack /m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/msbuild.binlog" - platform: Any CPU - configuration: $(BuildConfiguration) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - - script: dotnet build --no-restore -c $(BuildConfiguration) /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/build.binlog" displayName: dotnet build - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) - script: dotnet pack --no-build -c $(BuildConfiguration) /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/pack.binlog" displayName: dotnet pack - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) - task: DotNetCoreCLI@2 displayName: dotnet test -f net472 From 1a1917e0cdf8fb1afac30641c52b2866f334f58f Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 24 Feb 2021 09:57:51 -0700 Subject: [PATCH 17/99] Remove dead build authoring --- azure-pipelines/build.yml | 1 - azure-pipelines/publish-deployables.yml | 19 ------------------- 2 files changed, 20 deletions(-) delete mode 100644 azure-pipelines/publish-deployables.yml diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 764820fd..51e1fe31 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -65,4 +65,3 @@ jobs: - template: publish-codecoverage.yml parameters: includeMacOS: ${{ parameters.includeMacOS }} - - template: publish-deployables.yml diff --git a/azure-pipelines/publish-deployables.yml b/azure-pipelines/publish-deployables.yml deleted file mode 100644 index 9cb41fc6..00000000 --- a/azure-pipelines/publish-deployables.yml +++ /dev/null @@ -1,19 +0,0 @@ -steps: -- download: current - displayName: Download deployables - artifact: deployables-Windows - -- task: NuGetToolInstaller@1 - displayName: Use NuGet 5.x - inputs: - versionSpec: 5.x - -- task: NuGetCommand@2 - displayName: Push packages to CI feed - inputs: - command: push - packagesToPush: $(Pipeline.Workspace)/deployables-Windows/NuGet/*.nupkg - nuGetFeedType: internal - publishVstsFeed: $(ci_feed) - allowPackageConflicts: true - condition: and(succeeded(), eq(variables['push_to_ci'], 'true'), ne(variables['ci_feed'], ''), ne(variables['Build.Reason'], 'PullRequest')) From 70e32877551294b242d2cabb89ffb955944ffcb0 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 27 Apr 2021 08:18:59 -0600 Subject: [PATCH 18/99] Fix InsertPropsValues.ps1 --- azure-pipelines/variables/InsertPropsValues.ps1 | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/azure-pipelines/variables/InsertPropsValues.ps1 b/azure-pipelines/variables/InsertPropsValues.ps1 index 05df93b7..8d1e70a1 100644 --- a/azure-pipelines/variables/InsertPropsValues.ps1 +++ b/azure-pipelines/variables/InsertPropsValues.ps1 @@ -1,14 +1,2 @@ -$BinPath = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..\bin\Packages\$env:BUILDCONFIGURATION") - -$dirsToSearch = "$BinPath\NuGet\*.nupkg" |? { Test-Path $_ } -$icv=@() -if ($dirsToSearch) { - Get-ChildItem -Path $dirsToSearch |% { - if ($_.Name -match "^(.*)\.(\d+\.\d+\.\d+(?:-.*?)?)(?:\.symbols)?\.nupkg$") { - $id = $Matches[1] - $version = $Matches[2] - } - } -} - -Write-Output ([string]::join(',',$icv)) +# These values are commonly the same. +& "$PSScriptRoot/InsertConfigValues.ps1" From 4e2e5d5bd2d226055a91980eda9ad522602d4fa2 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 3 May 2021 09:23:38 -0600 Subject: [PATCH 19/99] Trust new Microsoft/nuget certificates --- nuget.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nuget.config b/nuget.config index 90f832a9..d9f432e1 100644 --- a/nuget.config +++ b/nuget.config @@ -18,9 +18,11 @@ Microsoft;xunit;manuel.roemer;sharwell;jamesnk;aarnott;MarcoRossignoli;Thecentury + + From 679db2a7343675e138cb42469f227fe335668ed5 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 12 May 2021 07:55:17 -0600 Subject: [PATCH 20/99] Handle 4-integer package versions in InsertConfigValues.ps1 --- azure-pipelines/variables/InsertConfigValues.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/variables/InsertConfigValues.ps1 b/azure-pipelines/variables/InsertConfigValues.ps1 index 1455a5b7..eaf00327 100644 --- a/azure-pipelines/variables/InsertConfigValues.ps1 +++ b/azure-pipelines/variables/InsertConfigValues.ps1 @@ -3,7 +3,7 @@ $BinPath = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..\bin\Packages\$env: $icv=@() if (Test-Path "$BinPath") { Get-ChildItem -Path "$BinPath\*.nupkg" -rec |% { - if ($_.Name -match "^(.*)\.(\d+\.\d+\.\d+(?:-.*?)?)(?:\.symbols)?\.nupkg$") { + if ($_.Name -match "^(.*?)\.(\d+\.\d+\.\d+(?:\.\d+)?(?:-.*?)?)(?:\.symbols)?\.nupkg$") { $id = $Matches[1] $version = $Matches[2] $icv += "$id=$version" From 936cffbc91ed3972ceb562d37eb0275f6c0172e3 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 21 Jun 2021 19:09:04 -0600 Subject: [PATCH 21/99] Workaround for microbuild plugin dependency bug --- init.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/init.ps1 b/init.ps1 index ca696265..c3c5b5e5 100755 --- a/init.ps1 +++ b/init.ps1 @@ -121,6 +121,11 @@ try { $EnvVars['LocLanguages'] = "JPN" } + # This is a workaround for https://dev.azure.com/devdiv/DevDiv/_workitems/edit/1283978 + if ($Signing -or $Setup -or $OptProf -or $Localization) { + & $InstallNuGetPkgScriptPath MicroBuild.Core.Sentinel -source $MicroBuildPackageSource -Verbosity $nugetVerbosity + } + & "$PSScriptRoot/tools/Set-EnvVars.ps1" -Variables $EnvVars | Out-Null } catch { From f1f2fce207a492f4a29ac292aa986dfca87db13d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 28 Jun 2021 08:40:45 -0600 Subject: [PATCH 22/99] Remove unused `BuildPlatform` build variable --- azure-pipelines/official.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index d00f13df..fb2ff576 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -37,7 +37,6 @@ stages: TreatWarningsAsErrors: true DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release - BuildPlatform: Any CPU push_to_ci: true NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages SignTypeSelection: ${{ parameters.SignTypeSelection }} From 671330c8bfdf3804d5db8932085863454d8c466a Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 28 Jun 2021 12:07:48 -0600 Subject: [PATCH 23/99] Only update config/props values on inserted packages --- azure-pipelines/artifacts/VSInsertion.ps1 | 2 +- azure-pipelines/variables/InsertConfigValues.ps1 | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines/artifacts/VSInsertion.ps1 b/azure-pipelines/artifacts/VSInsertion.ps1 index 9c9c4250..88cab0f8 100644 --- a/azure-pipelines/artifacts/VSInsertion.ps1 +++ b/azure-pipelines/artifacts/VSInsertion.ps1 @@ -8,7 +8,7 @@ if (!$BuildConfiguration) { $PackagesRoot = "$RepoRoot/bin/Packages/$BuildConfiguration/NuGet" -if (!(Test-Path $PackagesRoot)) { return } +if (!(Test-Path $PackagesRoot)) { return @{} } @{ "$PackagesRoot" = (Get-ChildItem $PackagesRoot -Recurse) diff --git a/azure-pipelines/variables/InsertConfigValues.ps1 b/azure-pipelines/variables/InsertConfigValues.ps1 index eaf00327..ddc472eb 100644 --- a/azure-pipelines/variables/InsertConfigValues.ps1 +++ b/azure-pipelines/variables/InsertConfigValues.ps1 @@ -1,9 +1,9 @@ -$BinPath = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..\bin\Packages\$env:BUILDCONFIGURATION") +$InsertedPkgs = (& "$PSScriptRoot\..\artifacts\VSInsertion.ps1") $icv=@() -if (Test-Path "$BinPath") { - Get-ChildItem -Path "$BinPath\*.nupkg" -rec |% { - if ($_.Name -match "^(.*?)\.(\d+\.\d+\.\d+(?:\.\d+)?(?:-.*?)?)(?:\.symbols)?\.nupkg$") { +foreach ($kvp in $InsertedPkgs.GetEnumerator()) { + $kvp.Value |% { + if ($_.Name -match "^(.*)\.(\d+\.\d+\.\d+(?:-.*?)?)(?:\.symbols)?\.nupkg$") { $id = $Matches[1] $version = $Matches[2] $icv += "$id=$version" From 5459e34d9f59e153637e2c8b49fa36a39265a5ef Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 1 Jul 2021 06:50:50 -0600 Subject: [PATCH 24/99] Use batched CI --- azure-pipelines/official.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index fb2ff576..d0199759 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -1,4 +1,5 @@ trigger: + batch: true branches: include: - main From 46334e874dd86736272896a33892183fdc447cf8 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 28 Jul 2021 13:23:59 -0600 Subject: [PATCH 25/99] Update microbuild feed URL to the newer format --- init.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init.ps1 b/init.ps1 index 3454b80a..825ce9fd 100755 --- a/init.ps1 +++ b/init.ps1 @@ -104,7 +104,7 @@ try { $InstallNuGetPkgScriptPath = ".\azure-pipelines\Install-NuGetPackage.ps1" $nugetVerbosity = 'quiet' if ($Verbose) { $nugetVerbosity = 'normal' } - $MicroBuildPackageSource = 'https://devdiv.pkgs.visualstudio.com/DefaultCollection/_packaging/MicroBuildToolset/nuget/v3/index.json' + $MicroBuildPackageSource = 'https://pkgs.dev.azure.com/devdiv/_packaging/MicroBuildToolset%40Local/nuget/v3/index.json' if ($Signing) { Write-Host "Installing MicroBuild signing plugin" -ForegroundColor $HeaderColor & $InstallNuGetPkgScriptPath MicroBuild.Plugins.Signing -source $MicroBuildPackageSource -Verbosity $nugetVerbosity From 70495e7db031ec433689a191f7ef6c7d31ad2ca1 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 28 Jul 2021 14:38:35 -0600 Subject: [PATCH 26/99] Remove obsolete task parameter --- azure-pipelines/microbuild.before.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index 206c1879..96cd3254 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -2,5 +2,4 @@ steps: - task: MicroBuildSigningPlugin@3 inputs: signType: $(SignType) - zipSources: false displayName: Install MicroBuild Signing Plugin From b1f3bdc6a226ee9385eb414ffcf2173cede3b313 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 6 Aug 2021 10:06:18 -0600 Subject: [PATCH 27/99] Enable Component Governance to PR builds (#111) It actually does something useful, and will soon be mandated. --- azure-pipelines/microbuild.after.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 853b6117..8559b27a 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -11,7 +11,6 @@ steps: - task: ComponentGovernanceComponentDetection@0 displayName: Component Detection - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - task: Ref12Analyze@0 displayName: Ref12 (Codex) Analyze From 9ce97e7830e218640dd01862305ac8b36d71aedc Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 6 Aug 2021 10:11:41 -0600 Subject: [PATCH 28/99] Remove workaround for microbuild plugin bug The bug is now fixed. --- init.ps1 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/init.ps1 b/init.ps1 index 825ce9fd..fef2cb09 100755 --- a/init.ps1 +++ b/init.ps1 @@ -129,11 +129,6 @@ try { $EnvVars['LocLanguages'] = "JPN" } - # This is a workaround for https://dev.azure.com/devdiv/DevDiv/_workitems/edit/1283978 - if ($Signing -or $Setup -or $OptProf -or $Localization) { - & $InstallNuGetPkgScriptPath MicroBuild.Core.Sentinel -source $MicroBuildPackageSource -Verbosity $nugetVerbosity - } - & "$PSScriptRoot/tools/Set-EnvVars.ps1" -Variables $EnvVars | Out-Null } catch { From 7e6aa52248c2b3bc5d466d39ec8518830e2b8991 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 8 Sep 2021 09:35:36 -0600 Subject: [PATCH 29/99] Switch to the VSEngSS-MicroBuild2019-1ES pool --- azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index d0199759..38ea7124 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -45,7 +45,7 @@ stages: jobs: - template: build.yml parameters: - windowsPool: VSEngSS-MicroBuild2019 + windowsPool: VSEngSS-MicroBuild2019-1ES includeMacOS: ${{ parameters.includeMacOS }} - stage: symbol_archive From cb6afb9d96ef72482684e0fcd4db5f5d75166920 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 8 Sep 2021 15:53:31 -0600 Subject: [PATCH 30/99] Update Microbuild version --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 7744e2b9..91ea7ecc 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -24,7 +24,7 @@ true snupkg - 2.0.65 + 2.0.66 From 341aba5d8f66353b57161a25248ddd3f1c172190 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 6 Oct 2021 11:36:13 -0600 Subject: [PATCH 31/99] Use public hosted pool for VS insertions We only need VSEng-ReleasePool for symbol archival. --- azure-pipelines/vs-insertion.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index 90645cb2..6aef7889 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -17,7 +17,8 @@ stages: displayName: VS insertion jobs: - deployment: insertion - pool: VSEng-ReleasePool + pool: + vmImage: windows-latest environment: No-Approval strategy: runOnce: From 8a53e8b57038869eb0b5d7cfc4a08650b4ca56e7 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 28 Oct 2021 07:13:05 -0600 Subject: [PATCH 32/99] Archive symbols on VSEng-ReleasePool-1ES --- azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 38ea7124..fc696565 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -53,7 +53,7 @@ stages: condition: and(succeeded(), eq(dependencies.Build.outputs['Windows.SetPipelineVariables.SignType'], 'Real')) jobs: - job: archive - pool: VSEng-ReleasePool + pool: VSEng-ReleasePool-1ES steps: - download: current artifact: Variables-Windows From c89a91562b6aeabbc18e6647165230d04a386d04 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 19 Nov 2021 12:44:29 -0700 Subject: [PATCH 33/99] Suppress microbuild nonshipping dependency --- test/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Directory.Build.props b/test/Directory.Build.props index ef88f73a..6ada957d 100644 --- a/test/Directory.Build.props +++ b/test/Directory.Build.props @@ -7,6 +7,6 @@ - + From 861b1285548438f448bed6bc6ac63161095cfbb4 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 19 Nov 2021 15:50:41 -0700 Subject: [PATCH 34/99] Build on Microbuild 2022 agents --- azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index fc696565..b6791c67 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -45,7 +45,7 @@ stages: jobs: - template: build.yml parameters: - windowsPool: VSEngSS-MicroBuild2019-1ES + windowsPool: VSEngSS-MicroBuild2022-1ES includeMacOS: ${{ parameters.includeMacOS }} - stage: symbol_archive From decffacc230f5a45d6cef669d0c81a08ce63353d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 22 Nov 2021 15:49:28 -0700 Subject: [PATCH 35/99] Define TeamEmail variable --- azure-pipelines/variables/TeamEmail.ps1 | 1 + 1 file changed, 1 insertion(+) create mode 100644 azure-pipelines/variables/TeamEmail.ps1 diff --git a/azure-pipelines/variables/TeamEmail.ps1 b/azure-pipelines/variables/TeamEmail.ps1 new file mode 100644 index 00000000..7cf66982 --- /dev/null +++ b/azure-pipelines/variables/TeamEmail.ps1 @@ -0,0 +1 @@ +'vsidemicrobuild@microsoft.com' From ca5dc779b4abe27cc6aca34a1498af6fa0a3eca9 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 2 Dec 2021 13:05:54 -0700 Subject: [PATCH 36/99] Generate NOTICE file and include in nuget packages --- Directory.Build.props | 4 ++++ azure-pipelines/microbuild.after.yml | 3 --- azure-pipelines/microbuild.before.yml | 9 +++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 70ab3c7b..ee246587 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -41,6 +41,10 @@ + + + + diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 8559b27a..70f07eaa 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -9,9 +9,6 @@ steps: condition: succeededOrFailed() displayName: MicroBuild Cleanup -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection - - task: Ref12Analyze@0 displayName: Ref12 (Codex) Analyze inputs: diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index 96cd3254..ab930694 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -1,4 +1,13 @@ steps: +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection + +- task: notice@0 + displayName: Generate NOTICE file + inputs: + outputfile: $(System.DefaultWorkingDirectory)/obj/NOTICE + outputformat: text + - task: MicroBuildSigningPlugin@3 inputs: signType: $(SignType) From 099d36f7c4fe6b82e091833675e52ede3617b7ca Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 2 Dec 2021 13:12:48 -0700 Subject: [PATCH 37/99] Fix PoliCheck to actually break builds on failures --- azure-pipelines/PoliCheckExclusions.xml | 10 ++++++++++ azure-pipelines/secure-development-tools.yml | 16 +++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 azure-pipelines/PoliCheckExclusions.xml diff --git a/azure-pipelines/PoliCheckExclusions.xml b/azure-pipelines/PoliCheckExclusions.xml new file mode 100644 index 00000000..5ae16710 --- /dev/null +++ b/azure-pipelines/PoliCheckExclusions.xml @@ -0,0 +1,10 @@ + + + NODE_MODULES|.STORE + + + + + + + diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index 941d5b0c..ca05939e 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -1,11 +1,17 @@ steps: - ### Check for checked in credentials. - task: CredScan@3 - displayName: 'Run CredScan' + displayName: Run CredScan - ### Run PoliCheck to check for disallowed terms. targetType: F indicates we're searching files and folders. -- task: PoliCheck@1 - displayName: 'Run PoliCheck' +- task: PoliCheck@2 + displayName: Run PoliCheck inputs: targetType: F + targetArgument: $(System.DefaultWorkingDirectory) + optionsUEPATH: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml + +- task: PostAnalysis@2 + displayName: Break on compliance issues + inputs: + GdnBreakAllTools: true + GdnBreakGdnToolBinSkimSeverity: Warning From a53b29ece9260d9d844cc4e90bd2ad84bcdf05cb Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 2 Dec 2021 13:18:59 -0700 Subject: [PATCH 38/99] Add SDT task reporting --- azure-pipelines/secure-development-tools.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index ca05939e..56d26279 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -10,6 +10,20 @@ steps: targetArgument: $(System.DefaultWorkingDirectory) optionsUEPATH: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml +- task: SdtReport@2 + displayName: Create Security Analysis Report + inputs: + GdnExportAllTools: true + +- task: PublishSecurityAnalysisLogs@3 + displayName: Publish Code Analysis Logs + inputs: + ArtifactName: CodeAnalysisLogs + ArtifactType: Container + PublishProcessedResults: true + AllTools: true + ToolLogsNotFoundAction: Standard + - task: PostAnalysis@2 displayName: Break on compliance issues inputs: From 848f9799a8d6b890195c9bc231766e4466349402 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 2 Dec 2021 13:19:14 -0700 Subject: [PATCH 39/99] Add BinSkim to SDT tasks --- azure-pipelines/Get-SymbolFiles.ps1 | 10 ++++++++-- azure-pipelines/secure-development-tools.yml | 7 +++++++ azure-pipelines/variables/BinSkimTargets.ps1 | 4 ++++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 azure-pipelines/variables/BinSkimTargets.ps1 diff --git a/azure-pipelines/Get-SymbolFiles.ps1 b/azure-pipelines/Get-SymbolFiles.ps1 index 45afe7c2..8793e19c 100644 --- a/azure-pipelines/Get-SymbolFiles.ps1 +++ b/azure-pipelines/Get-SymbolFiles.ps1 @@ -5,12 +5,16 @@ The root path to recursively search for PDBs. .PARAMETER Tests A switch indicating to find test-related PDBs instead of product-only PDBs. +.PARAMETER ConvertToWindowsPDBs + A switch to convert and return paths to Windows PDBs instead of portable PDBs. + Ignored on non-Windows agents. #> [CmdletBinding()] param ( [parameter(Mandatory=$true)] [string]$Path, - [switch]$Tests + [switch]$Tests, + [switch]$ConvertToWindowsPDBs=$true ) $WindowsPdbSubDirName = "symstore" @@ -54,7 +58,7 @@ $PDBs |% { Write-Output $BinaryImagePath - if (-not ($IsMacOS -or $IsLinux)) { + if ($ConvertToWindowsPDBs -and -not ($IsMacOS -or $IsLinux)) { # Convert the PDB to legacy Windows PDBs Write-Host "Converting PDB for $_" -ForegroundColor DarkGray $WindowsPdbDir = "$($_.Directory.FullName)\$WindowsPdbSubDirName" @@ -65,5 +69,7 @@ $PDBs |% { } Write-Output "$WindowsPdbDir\$($_.BaseName).pdb" + } else { + Write-Output $_.FullName } } diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index 56d26279..0ac9b324 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -10,6 +10,13 @@ steps: targetArgument: $(System.DefaultWorkingDirectory) optionsUEPATH: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml +- task: BinSkim@3 + displayName: Run BinSkim + inputs: + InputType: Basic + Function: analyze + AnalyzeTarget: $(BinSkimTargets) + - task: SdtReport@2 displayName: Create Security Analysis Report inputs: diff --git a/azure-pipelines/variables/BinSkimTargets.ps1 b/azure-pipelines/variables/BinSkimTargets.ps1 new file mode 100644 index 00000000..5c0dd24e --- /dev/null +++ b/azure-pipelines/variables/BinSkimTargets.ps1 @@ -0,0 +1,4 @@ +$Path = "$PSScriptRoot\..\..\bin" +if (Test-Path $Path) { + [string]::join(';', (& "$PSScriptRoot\..\Get-SymbolFiles.ps1" -ConvertToWindowsPDBs:$false -Path $Path)) +} From 02a6d80f2ca20040fc1d3624e461c2f40b5773d0 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 2 Dec 2021 13:27:49 -0700 Subject: [PATCH 40/99] Add APIScan SDT task --- azure-pipelines/build.yml | 9 +++++ azure-pipelines/falsepositives.gdnsuppress | 12 +++++++ azure-pipelines/microbuild.after.yml | 6 ++++ azure-pipelines/official.yml | 5 +++ azure-pipelines/secure-development-tools.yml | 35 ++++++++++++++++++++ 5 files changed, 67 insertions(+) create mode 100644 azure-pipelines/falsepositives.gdnsuppress diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 58cb9716..8523ab85 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -4,10 +4,17 @@ parameters: default: vmImage: windows-2022 - name: includeMacOS +- name: EnableAPIScan + type: boolean + default: false jobs: - job: Windows pool: ${{ parameters.windowsPool }} + variables: + - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + # https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/25351/APIScan-step-by-step-guide-to-setting-up-a-Pipeline + - group: VSCloudServices-APIScan steps: - checkout: self clean: true @@ -27,6 +34,8 @@ jobs: - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - template: microbuild.after.yml + parameters: + EnableAPIScan: ${{ parameters.EnableAPIScan }} - template: expand-template.yml diff --git a/azure-pipelines/falsepositives.gdnsuppress b/azure-pipelines/falsepositives.gdnsuppress new file mode 100644 index 00000000..1248172b --- /dev/null +++ b/azure-pipelines/falsepositives.gdnsuppress @@ -0,0 +1,12 @@ +{ + "version": "latest", + "suppressionSets": { + "falsepositives": { + "name": "falsepositives", + "createdDate": "2021-12-03 00:23:08Z", + "lastUpdatedDate": "2021-12-03 00:23:08Z" + } + }, + "results": { + } +} diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 70f07eaa..6f107833 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -1,3 +1,7 @@ +parameters: +- name: EnableAPIScan + type: boolean + steps: - task: MicroBuildCodesignVerify@3 displayName: Verify Signed Files @@ -23,3 +27,5 @@ steps: continueOnError: true - template: secure-development-tools.yml + parameters: + EnableAPIScan: ${{ parameters.EnableAPIScan }} diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index b6791c67..bac667b6 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -27,6 +27,10 @@ parameters: displayName: Build on macOS type: boolean default: false # macOS is often bogged down in Azure Pipelines +- name: EnableAPIScan + displayName: Run APIScan + type: boolean + default: true variables: NugetSecurityAnalysisWarningLevel: none # nuget.config requires signed packages by trusted owners @@ -45,6 +49,7 @@ stages: jobs: - template: build.yml parameters: + EnableAPIScan: ${{ parameters.EnableAPIScan }} windowsPool: VSEngSS-MicroBuild2022-1ES includeMacOS: ${{ parameters.includeMacOS }} diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index 0ac9b324..e6b4d624 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -1,3 +1,7 @@ +parameters: +- name: EnableAPIScan + type: boolean + steps: - task: CredScan@3 @@ -17,6 +21,27 @@ steps: Function: analyze AnalyzeTarget: $(BinSkimTargets) +- task: CopyFiles@2 + displayName: Collect APIScan inputs + inputs: + SourceFolder: $(Build.ArtifactStagingDirectory)/Symbols-$(Agent.JobName) + # Exclude any patterns from the Contents (e.g. `!**/git2*`) that we have symbols for but do not need to run APIScan on. + Contents: | + ** + TargetFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs + +- task: APIScan@2 + displayName: Run APIScan + inputs: + softwareFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs + softwareName: $(SymbolsFeatureName) + softwareVersionNum: $(GitBuildVersion) + isLargeApp: false + toolVersion: Latest + condition: and(succeeded(), ${{ parameters.EnableAPIScan }}, ne(variables.ApiScanClientId, '')) + env: + AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) + - task: SdtReport@2 displayName: Create Security Analysis Report inputs: @@ -36,3 +61,13 @@ steps: inputs: GdnBreakAllTools: true GdnBreakGdnToolBinSkimSeverity: Warning + GdnBreakSuppressionFiles: $(System.DefaultWorkingDirectory)/azure-pipelines/falsepositives.gdnsuppress + GdnBreakSuppressionSets: falsepositives + GdnBreakOutputSuppressionFile: $(Build.ArtifactStagingDirectory)/guardian_failures_as_suppressions/ + GdnBreakOutputSuppressionSet: falsepositives + +# This is useful when false positives appear so we can copy some of the output into the suppressions file. +- publish: $(Build.ArtifactStagingDirectory)/guardian_failures_as_suppressions + artifact: guardian_failures_as_suppressions + displayName: Publish Guardian failures + condition: failed() From 7d02faa9b0662aa263dbd2cf068e473b2e911c8e Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 3 Dec 2021 08:07:31 -0700 Subject: [PATCH 41/99] Skip copying APIScan inputs when APIScan is disabled --- azure-pipelines/secure-development-tools.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index e6b4d624..21eab9d3 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -29,6 +29,7 @@ steps: Contents: | ** TargetFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs + condition: and(succeeded(), ${{ parameters.EnableAPIScan }}, ne(variables.ApiScanClientId, '')) - task: APIScan@2 displayName: Run APIScan From c2b0252db640401aa5b1cf62f44d30e4298bff46 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 3 Dec 2021 11:48:23 -0700 Subject: [PATCH 42/99] Reduce frequency of changing version given to APIScan --- azure-pipelines/build.yml | 2 +- azure-pipelines/secure-development-tools.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 8523ab85..27ba5b5b 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -24,7 +24,7 @@ jobs: - template: install-dependencies.yml - - powershell: '& (./azure-pipelines/Get-nbgv.ps1) cloud -c' + - powershell: '& (./azure-pipelines/Get-nbgv.ps1) cloud -ca' displayName: Set build number - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index 21eab9d3..99925c13 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -36,7 +36,7 @@ steps: inputs: softwareFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs softwareName: $(SymbolsFeatureName) - softwareVersionNum: $(GitBuildVersion) + softwareVersionNum: $(NBGV_MajorMinorVersion) isLargeApp: false toolVersion: Latest condition: and(succeeded(), ${{ parameters.EnableAPIScan }}, ne(variables.ApiScanClientId, '')) From 4f87de906bc58b5b6a505c4eadbd7ff6068ac60a Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 6 Jan 2022 12:39:52 -0700 Subject: [PATCH 43/99] Bump BinSkim to v4 --- azure-pipelines/secure-development-tools.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index 99925c13..c5ebf04e 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -14,12 +14,13 @@ steps: targetArgument: $(System.DefaultWorkingDirectory) optionsUEPATH: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml -- task: BinSkim@3 +- task: BinSkim@4 displayName: Run BinSkim inputs: InputType: Basic Function: analyze - AnalyzeTarget: $(BinSkimTargets) + TargetPattern: guardianGlob + AnalyzeTargetGlob: $(BinSkimTargets) - task: CopyFiles@2 displayName: Collect APIScan inputs From 6235412ac9f7bb02c49be49821f329fc30d7499f Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 13 Jan 2022 11:21:05 -0700 Subject: [PATCH 44/99] Build SBOM manifest as part of build --- Expand-Template.ps1 | 3 +++ azure-pipelines/artifacts/VSInsertion.ps1 | 3 +++ azure-pipelines/build.yml | 5 +++++ azure-pipelines/microbuild.after.yml | 9 +++++++++ azure-pipelines/official.yml | 1 + 5 files changed, 21 insertions(+) diff --git a/Expand-Template.ps1 b/Expand-Template.ps1 index 8d4c2f4b..5705e722 100755 --- a/Expand-Template.ps1 +++ b/Expand-Template.ps1 @@ -155,6 +155,9 @@ try { $YmlReplacements['(codecov_token: ).*(#.*)'] = "#`$1`$2" } Replace-Placeholders -Path "azure-pipelines.yml" -Replacements $YmlReplacements + Replace-Placeholders -Path "azure-pipelines/microbuild.after.yml" -Replacements @{ + 'Library' = $LibraryName; + } Replace-Placeholders -Path "azure-pipelines/variables/InsertVersionsValues.ps1" -Replacements @{ 'LibraryName' = $LibraryName; diff --git a/azure-pipelines/artifacts/VSInsertion.ps1 b/azure-pipelines/artifacts/VSInsertion.ps1 index 88cab0f8..80e621e7 100644 --- a/azure-pipelines/artifacts/VSInsertion.ps1 +++ b/azure-pipelines/artifacts/VSInsertion.ps1 @@ -8,6 +8,9 @@ if (!$BuildConfiguration) { $PackagesRoot = "$RepoRoot/bin/Packages/$BuildConfiguration/NuGet" +# This artifact is not ready if we're running on the devdiv AzDO account and we don't have an SBOM yet. +if ($env:SYSTEM_COLLECTIONID -eq '011b8bdf-6d56-4f87-be0d-0092136884d9' -and -not (Test-Path $PackagesRoot/_manifest)) { return @{} } + if (!(Test-Path $PackagesRoot)) { return @{} } @{ diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 27ba5b5b..1d003eb1 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -36,6 +36,11 @@ jobs: - template: microbuild.after.yml parameters: EnableAPIScan: ${{ parameters.EnableAPIScan }} + # Repeat this step to scoop up any artifacts that would only be collected after running microbuild.after.yml + - powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)" + failOnStderr: true + displayName: Publish artifacts + condition: succeededOrFailed() - template: expand-template.yml diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 6f107833..e889a464 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -13,6 +13,15 @@ steps: condition: succeededOrFailed() displayName: MicroBuild Cleanup +- task: ManifestGeneratorTask@0 + displayName: Software Bill of Materials generation + inputs: + BuildDropPath: $(System.DefaultWorkingDirectory)/bin/Library/$(BuildConfiguration) + BuildComponentPath: $(System.DefaultWorkingDirectory)/obj/src/Library + +- powershell: Copy-Item -Recurse "$(System.DefaultWorkingDirectory)/bin/Library/$(BuildConfiguration)/_manifest" "$(System.DefaultWorkingDirectory)/bin/Packages/$(BuildConfiguration)/NuGet" + displayName: Publish Software Bill of Materials + - task: Ref12Analyze@0 displayName: Ref12 (Codex) Analyze inputs: diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index bac667b6..3fca0999 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -45,6 +45,7 @@ stages: push_to_ci: true NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages SignTypeSelection: ${{ parameters.SignTypeSelection }} + Packaging.EnableSBOMSigning: true jobs: - template: build.yml From d581062a43eafc81b1320c18656d2d1eb9519ca2 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 13 Jan 2022 16:06:27 -0700 Subject: [PATCH 45/99] Retain inserted builds for two years --- azure-pipelines/vs-insertion.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index 6aef7889..9f09b8ef 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -49,3 +49,12 @@ stages: AutoCompleteMergeStrategy: Squash - task: MicroBuildCleanup@1 displayName: Send Telemetry + - powershell: | + $contentType = 'application/json'; + $headers = @{ Authorization = 'Bearer $(System.AccessToken)' }; + $rawRequest = @{ daysValid = 365 * 2; definitionId = $(resources.pipeline.CI.pipelineID); ownerId = 'User:$(Build.RequestedForId)'; protectPipeline = $false; runId = $(resources.pipeline.CI.runId) }; + $request = ConvertTo-Json @($rawRequest); + Write-Host $request + $uri = "$(System.CollectionUri)$(System.TeamProject)/_apis/build/retention/leases?api-version=6.0-preview.1"; + Invoke-RestMethod -uri $uri -method POST -Headers $headers -ContentType $contentType -Body $request; + displayName: Retain inserted builds From d2aa6efa1e59faa9a36e6eec093ced72fe62e8b1 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 21 Jan 2022 08:01:13 -0700 Subject: [PATCH 46/99] Fix symbol archival step Symbol archival was failing when InsertConfigValues.ps1 was running from a downloaded artifact, but it is only designed to run within the repo. It failed within the repo because VSInsertion.ps1 started returning an empty set of files due to the recent SBOM change in 6235412: Build SBOM manifest as part of build The fix is to give the scripts a way to communicate that the SBOM is not relevant so that InsertConfigValues can still run when required. --- azure-pipelines/artifacts/VSInsertion.ps1 | 10 +++++++++- azure-pipelines/release-deployment-prep.yml | 2 +- azure-pipelines/variables/InsertConfigValues.ps1 | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/azure-pipelines/artifacts/VSInsertion.ps1 b/azure-pipelines/artifacts/VSInsertion.ps1 index 80e621e7..4c56deab 100644 --- a/azure-pipelines/artifacts/VSInsertion.ps1 +++ b/azure-pipelines/artifacts/VSInsertion.ps1 @@ -1,5 +1,13 @@ # This artifact captures everything needed to insert into VS (NuGet packages, insertion metadata, etc.) +<# +.PARAMETER SbomNotRequired + Indicates that returning the artifacts available is preferable to nothing at all when the SBOM has not yet been generated. +#> +Param ( + [switch]$SbomNotRequired +) + $RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..") $BuildConfiguration = $env:BUILDCONFIGURATION if (!$BuildConfiguration) { @@ -9,7 +17,7 @@ if (!$BuildConfiguration) { $PackagesRoot = "$RepoRoot/bin/Packages/$BuildConfiguration/NuGet" # This artifact is not ready if we're running on the devdiv AzDO account and we don't have an SBOM yet. -if ($env:SYSTEM_COLLECTIONID -eq '011b8bdf-6d56-4f87-be0d-0092136884d9' -and -not (Test-Path $PackagesRoot/_manifest)) { return @{} } +if ($env:SYSTEM_COLLECTIONID -eq '011b8bdf-6d56-4f87-be0d-0092136884d9' -and -not (Test-Path $PackagesRoot/_manifest) -and -not $SbomNotRequired) { return @{} } if (!(Test-Path $PackagesRoot)) { return @{} } diff --git a/azure-pipelines/release-deployment-prep.yml b/azure-pipelines/release-deployment-prep.yml index 6dee28e5..059fd1d0 100644 --- a/azure-pipelines/release-deployment-prep.yml +++ b/azure-pipelines/release-deployment-prep.yml @@ -3,7 +3,7 @@ steps: artifact: Variables-Windows displayName: Download Variables-Windows artifact - task: PowerShell@2 - displayName: Set VSTS variables based on artifacts + displayName: Set pipeline variables based on artifacts inputs: targetType: filePath filePath: $(Pipeline.Workspace)/CI/Variables-Windows/_pipelines.ps1 diff --git a/azure-pipelines/variables/InsertConfigValues.ps1 b/azure-pipelines/variables/InsertConfigValues.ps1 index ddc472eb..efed018c 100644 --- a/azure-pipelines/variables/InsertConfigValues.ps1 +++ b/azure-pipelines/variables/InsertConfigValues.ps1 @@ -1,4 +1,4 @@ -$InsertedPkgs = (& "$PSScriptRoot\..\artifacts\VSInsertion.ps1") +$InsertedPkgs = (& "$PSScriptRoot\..\artifacts\VSInsertion.ps1" -SbomNotRequired) $icv=@() foreach ($kvp in $InsertedPkgs.GetEnumerator()) { From d043d36952bfd1e79428ca0657a98ff38b7813a2 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 2 Feb 2022 21:50:53 -0700 Subject: [PATCH 47/99] Update MicroBuild version --- Directory.Build.props | 2 +- init.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 65219a6f..aef458c1 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -28,7 +28,7 @@ true snupkg - 2.0.66 + 2.0.68 diff --git a/init.ps1 b/init.ps1 index 043d4551..0a7e1cbd 100755 --- a/init.ps1 +++ b/init.ps1 @@ -114,7 +114,7 @@ try { if ($Setup) { Write-Host "Installing MicroBuild SwixBuild plugin..." -ForegroundColor $HeaderColor - & $InstallNuGetPkgScriptPath MicroBuild.Plugins.SwixBuild -source $MicroBuildPackageSource -Verbosity $nugetVerbosity + & $InstallNuGetPkgScriptPath Microsoft.VisualStudioEng.MicroBuild.Plugins.SwixBuild -source $MicroBuildPackageSource -Verbosity $nugetVerbosity } if ($OptProf) { From e261c40938f2a1190824fbec4c7c020292f5f5b3 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 17 Feb 2022 09:58:44 -0700 Subject: [PATCH 48/99] Switch to EO-compliant agent pools --- azure-pipelines/release.yml | 6 ++++-- azure-pipelines/vs-insertion.yml | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index 0ef91cb3..1470f765 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -15,7 +15,8 @@ stages: jobs: - deployment: create pool: - vmImage: ubuntu-latest + name: AzurePipelines-EO + vmImage: AzurePipelinesUbuntu20.04compliant environment: No-Approval strategy: runOnce: @@ -49,7 +50,8 @@ stages: jobs: - deployment: push pool: - vmImage: ubuntu-latest + name: AzurePipelines-EO + vmImage: AzurePipelinesUbuntu20.04compliant environment: No-Approval strategy: runOnce: diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index 9f09b8ef..8c5b1aa9 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -18,7 +18,8 @@ stages: jobs: - deployment: insertion pool: - vmImage: windows-latest + name: AzurePipelines-EO + vmImage: AzurePipelinesWindows2022compliant environment: No-Approval strategy: runOnce: From 7d468a2f3f06c9e7ea86e34dca1eecdfc3e39780 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 25 Feb 2022 07:01:25 -0700 Subject: [PATCH 49/99] Fix VS insertion pool --- azure-pipelines/vs-insertion.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index 8c5b1aa9..77853a6a 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -17,9 +17,7 @@ stages: displayName: VS insertion jobs: - deployment: insertion - pool: - name: AzurePipelines-EO - vmImage: AzurePipelinesWindows2022compliant + pool: VSEngSS-MicroBuild2022-1ES environment: No-Approval strategy: runOnce: From 8a61e11c513a79b1f5ab8f747cc4414dba4946ee Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 9 Mar 2022 18:44:12 -0700 Subject: [PATCH 50/99] Delete a remnant of push_to_ci --- azure-pipelines/official.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index a4ae3884..1c800f3c 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -42,7 +42,6 @@ stages: TreatWarningsAsErrors: true DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release - push_to_ci: true NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages SignTypeSelection: ${{ parameters.SignTypeSelection }} Packaging.EnableSBOMSigning: true From 4feafe4c1c30128f97780fa78c7dc271b2bb80c7 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 31 Mar 2022 13:33:50 -0600 Subject: [PATCH 51/99] Add sample azure-public stage --- azure-pipelines/official.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 1c800f3c..d7b949d3 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -60,6 +60,7 @@ stages: - job: archive pool: VSEng-ReleasePool-1ES steps: + - checkout: none - download: current artifact: Variables-Windows displayName: Download Variables-Windows artifact @@ -81,3 +82,29 @@ stages: SymbolsAgentPath: $(Pipeline.Workspace)/symbols-legacy - task: MicroBuildCleanup@1 displayName: Send Telemetry + +# - stage: azure_public_vsimpl_feed +# displayName: azure-public/vs-impl feed +# condition: and(succeeded(), eq(dependencies.Build.outputs['Windows.SetPipelineVariables.SignType'], 'Real')) +# jobs: +# - job: push +# pool: +# name: AzurePipelines-EO +# vmImage: AzurePipelinesUbuntu20.04compliant +# steps: +# - checkout: none +# - download: current +# artifact: deployables-Windows +# displayName: Download deployables-Windows artifact +# - task: UseDotNet@2 +# displayName: Install .NET SDK +# inputs: +# packageType: sdk +# version: 6.x +# - task: NuGetAuthenticate@0 +# displayName: Authenticate NuGet feeds +# inputs: +# nuGetServiceConnections: azure-public/vs-impl +# forceReinstallCredentialProvider: true +# - script: dotnet nuget push $(Pipeline.Workspace)/deployables-Windows/NuGet/*.nupkg -s https://pkgs.dev.azure.com/azure-public/vside/_packaging/vs-impl/nuget/v3/index.json --api-key azdo --skip-duplicate +# displayName: Push nuget packages From e899f99b887eb091e4068958078638fa2bcad9b8 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 11 Apr 2022 13:57:10 -0600 Subject: [PATCH 52/99] Do not trigger CI on changes to the release or vs-insertion pipeline --- azure-pipelines/official.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index d7b949d3..48ef6cc9 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -10,6 +10,8 @@ trigger: - doc/ - '*.md' - .vscode/ + - azure-pipelines/release.yml + - azure-pipelines/vs-insertion.yml #schedules: #- cron: "0 3 * * *" # Daily @ 8 PM PST # displayName: Daily vs-insertion From 885c8ec6e85fb70460a35cad77e1a46934567135 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 12 Apr 2022 06:02:02 -0600 Subject: [PATCH 53/99] Turn off SBOM signing It times out too frequently. --- azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 48ef6cc9..b13d2e1d 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -46,7 +46,7 @@ stages: BuildConfiguration: Release NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages SignTypeSelection: ${{ parameters.SignTypeSelection }} - Packaging.EnableSBOMSigning: true + Packaging.EnableSBOMSigning: false jobs: - template: build.yml From 33e19c8934b26a2508f79353a5c20ba259a9dcac Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 12 Apr 2022 08:29:59 -0600 Subject: [PATCH 54/99] Improve pipeline verbosity --- azure-pipelines/Convert-PDB.ps1 | 57 +++++++++++------------ azure-pipelines/artifacts/VSInsertion.ps1 | 17 ++++++- azure-pipelines/microbuild.after.yml | 2 +- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/azure-pipelines/Convert-PDB.ps1 b/azure-pipelines/Convert-PDB.ps1 index 2d394e72..f119a164 100644 --- a/azure-pipelines/Convert-PDB.ps1 +++ b/azure-pipelines/Convert-PDB.ps1 @@ -8,36 +8,35 @@ .PARAMETER OutputPath The path of the output PDB to write. #> -#Function Convert-PortableToWindowsPDB() { - Param( - [Parameter(Mandatory=$true,Position=0)] - [string]$DllPath, - [Parameter()] - [string]$PdbPath, - [Parameter(Mandatory=$true,Position=1)] - [string]$OutputPath - ) +[CmdletBinding()] +Param( + [Parameter(Mandatory=$true,Position=0)] + [string]$DllPath, + [Parameter()] + [string]$PdbPath, + [Parameter(Mandatory=$true,Position=1)] + [string]$OutputPath +) - if ($IsMacOS -or $IsLinux) { - Write-Error "This script only works on Windows" - return - } +if ($IsMacOS -or $IsLinux) { + Write-Error "This script only works on Windows" + return +} - $version = '1.1.0-beta2-21101-01' - $baseDir = "$PSScriptRoot/../obj/tools" - $pdb2pdbpath = "$baseDir/Microsoft.DiaSymReader.Pdb2Pdb.$version/tools/Pdb2Pdb.exe" - if (-not (Test-Path $pdb2pdbpath)) { - if (-not (Test-Path $baseDir)) { New-Item -Type Directory -Path $baseDir | Out-Null } - $baseDir = (Resolve-Path $baseDir).Path # Normalize it - Write-Verbose "& (& $PSScriptRoot/Get-NuGetTool.ps1) install Microsoft.DiaSymReader.Pdb2Pdb -version $version -PackageSaveMode nuspec -OutputDirectory $baseDir -Source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json | Out-Null" - & (& $PSScriptRoot/Get-NuGetTool.ps1) install Microsoft.DiaSymReader.Pdb2Pdb -version $version -PackageSaveMode nuspec -OutputDirectory $baseDir -Source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json | Out-Null - } +$version = '1.1.0-beta2-21101-01' +$baseDir = "$PSScriptRoot/../obj/tools" +$pdb2pdbpath = "$baseDir/Microsoft.DiaSymReader.Pdb2Pdb.$version/tools/Pdb2Pdb.exe" +if (-not (Test-Path $pdb2pdbpath)) { + if (-not (Test-Path $baseDir)) { New-Item -Type Directory -Path $baseDir | Out-Null } + $baseDir = (Resolve-Path $baseDir).Path # Normalize it + Write-Verbose "& (& $PSScriptRoot/Get-NuGetTool.ps1) install Microsoft.DiaSymReader.Pdb2Pdb -version $version -PackageSaveMode nuspec -OutputDirectory $baseDir -Source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json | Out-Null" + & (& $PSScriptRoot/Get-NuGetTool.ps1) install Microsoft.DiaSymReader.Pdb2Pdb -version $version -PackageSaveMode nuspec -OutputDirectory $baseDir -Source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json | Out-Null +} - $args = $DllPath,'/out',$OutputPath,'/nowarn','0021' - if ($PdbPath) { - $args += '/pdb',$PdbPath - } +$args = $DllPath,'/out',$OutputPath,'/nowarn','0021' +if ($PdbPath) { + $args += '/pdb',$PdbPath +} - Write-Verbose "$pdb2pdbpath $args" - & $pdb2pdbpath $args -#} +Write-Verbose "$pdb2pdbpath $args" +& $pdb2pdbpath $args diff --git a/azure-pipelines/artifacts/VSInsertion.ps1 b/azure-pipelines/artifacts/VSInsertion.ps1 index 4c56deab..0d1cb373 100644 --- a/azure-pipelines/artifacts/VSInsertion.ps1 +++ b/azure-pipelines/artifacts/VSInsertion.ps1 @@ -4,10 +4,17 @@ .PARAMETER SbomNotRequired Indicates that returning the artifacts available is preferable to nothing at all when the SBOM has not yet been generated. #> +[CmdletBinding()] Param ( [switch]$SbomNotRequired ) +if ($IsMacOS -or $IsLinux) { + # We only package up for insertions on Windows agents since they are where optprof can happen. + Write-Verbose "Skipping VSInsertion artifact since we're not on Windows." + return @{} +} + $RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..") $BuildConfiguration = $env:BUILDCONFIGURATION if (!$BuildConfiguration) { @@ -17,9 +24,15 @@ if (!$BuildConfiguration) { $PackagesRoot = "$RepoRoot/bin/Packages/$BuildConfiguration/NuGet" # This artifact is not ready if we're running on the devdiv AzDO account and we don't have an SBOM yet. -if ($env:SYSTEM_COLLECTIONID -eq '011b8bdf-6d56-4f87-be0d-0092136884d9' -and -not (Test-Path $PackagesRoot/_manifest) -and -not $SbomNotRequired) { return @{} } +if ($env:SYSTEM_COLLECTIONID -eq '011b8bdf-6d56-4f87-be0d-0092136884d9' -and -not (Test-Path $PackagesRoot/_manifest) -and -not $SbomNotRequired) { + Write-Host "Skipping because SBOM isn't generated yet." + return @{} +} -if (!(Test-Path $PackagesRoot)) { return @{} } +if (!(Test-Path $PackagesRoot)) { + Write-Warning "Skipping because packages haven't been built yet." + return @{} +} @{ "$PackagesRoot" = (Get-ChildItem $PackagesRoot -Recurse) diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index e889a464..87634808 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -19,7 +19,7 @@ steps: BuildDropPath: $(System.DefaultWorkingDirectory)/bin/Library/$(BuildConfiguration) BuildComponentPath: $(System.DefaultWorkingDirectory)/obj/src/Library -- powershell: Copy-Item -Recurse "$(System.DefaultWorkingDirectory)/bin/Library/$(BuildConfiguration)/_manifest" "$(System.DefaultWorkingDirectory)/bin/Packages/$(BuildConfiguration)/NuGet" +- powershell: Copy-Item -Recurse -Verbose "$(System.DefaultWorkingDirectory)/bin/Library/$(BuildConfiguration)/_manifest" "$(System.DefaultWorkingDirectory)/bin/Packages/$(BuildConfiguration)/NuGet" displayName: Publish Software Bill of Materials - task: Ref12Analyze@0 From cbdabad295480b1844ad94a9ae64ac86d6b5a8a9 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 14 Apr 2022 07:09:32 -0600 Subject: [PATCH 55/99] Customize documented steps for microbuild branch --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a80b7b2d..cd9a4d2f 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ The best way to keep your repo in sync with this template's evolving features an git checkout main # your default branch git pull # make sure you're at tip git fetch libtemplate # fetch latest Library.Template -git merge libtemplate/main +git merge libtemplate/microbuild ``` There will frequently be merge conflicts to work out, but they will be easier to resolve than running the `Apply-Template.ps1` script every time, which simply blows away all your local changes with the latest from the template. @@ -74,7 +74,7 @@ Use `git rev-parse HEAD` within the Library.Template repo and record the resulti Run the `Apply-Template.ps1` script, passing in the path to your own Library.Template-based repo. This will blow away most customizations you may have made to your repo's build authoring. You should *carefully* review all changes to your repo, staging those changes that you want to keep and reverting those that remove customizations you made. Now it's time to commit your changes. We do this in a very low-level way in order to have git record this as a *merge* commit even though it didn't start as a merge. -By doing this, git will allow future merges from `libtemplate/main` and only new changes will be brought down, which will be much easier than the `Apply-Template.ps1` script you just ran. +By doing this, git will allow future merges from `libtemplate/microbuild` and only new changes will be brought down, which will be much easier than the `Apply-Template.ps1` script you just ran. We create the merge commit with these commands: 1. Be sure to have staged or reverted all the changes in your repo. From e0d4ad51a33eacceb0ef1c20a43f7a8b75c79c9d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sun, 1 May 2022 21:35:19 -0600 Subject: [PATCH 56/99] Add support for running 32-bit tests --- azure-pipelines/dotnet-test-cloud.ps1 | 45 ++++++++-- tools/Install-DotNetSdk.ps1 | 124 ++++++++++++++++++++------ 2 files changed, 138 insertions(+), 31 deletions(-) diff --git a/azure-pipelines/dotnet-test-cloud.ps1 b/azure-pipelines/dotnet-test-cloud.ps1 index 877f11ed..43cdc3e0 100755 --- a/azure-pipelines/dotnet-test-cloud.ps1 +++ b/azure-pipelines/dotnet-test-cloud.ps1 @@ -1,15 +1,50 @@ #!/usr/bin/env pwsh +<# +.SYNOPSIS + Runs tests as they are run in cloud test runs. +.PARAMETER Configuration + The configuration within which to run tests +.PARAMETER Agent + The name of the agent. This is used in preparing test run titles. +.PARAMETER PublishResults + A switch to publish results to Azure Pipelines. +.PARAMETER x86 + A switch to run the tests in an x86 process. +.PARAMETER dotnet32 + The path to a 32-bit dotnet executable to use. +#> +[CmdletBinding()] Param( [string]$Configuration='Debug', [string]$Agent='Local', - [switch]$PublishResults + [switch]$PublishResults, + [switch]$x86, + [string]$dotnet32 ) $RepoRoot = (Resolve-Path "$PSScriptRoot/..").Path $ArtifactStagingFolder = & "$PSScriptRoot/Get-ArtifactsStagingDirectory.ps1" -dotnet test $RepoRoot ` +$dotnet = 'dotnet' +if ($x86) { + $x86RunTitleSuffix = ", x86" + if ($dotnet32) { + $dotnet = $dotnet32 + } else { + $dotnet32Possibilities = "$PSScriptRoot\../obj/tools/x86/.dotnet/dotnet.exe", "$env:AGENT_TOOLSDIRECTORY/x86/dotnet/dotnet.exe", "${env:ProgramFiles(x86)}\dotnet\dotnet.exe" + $dotnet32Matches = $dotnet32Possibilities |? { Test-Path $_ } + if ($dotnet32Matches) { + $dotnet = Resolve-Path @($dotnet32Matches)[0] + Write-Host "Running tests using `"$dotnet`"" -ForegroundColor DarkGray + } else { + Write-Error "Unable to find 32-bit dotnet.exe" + return 1 + } + } +} + +& $dotnet test $RepoRoot ` --no-build ` -c $Configuration ` --filter "TestCategory!=FailsInCloudTest" ` @@ -18,7 +53,7 @@ dotnet test $RepoRoot ` --blame-crash ` -bl:"$ArtifactStagingFolder/build_logs/test.binlog" ` --diag "$ArtifactStagingFolder/test_logs/diag.log;TraceLevel=info" ` - --logger trx + --logger trx ` $unknownCounter = 0 Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx |% { @@ -33,13 +68,13 @@ Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx |% { if ($matches.rid) { $runTitle = "$($matches.lib) ($($matches.tfm), $($matches.rid), $Agent)" } else { - $runTitle = "$($matches.lib) ($($matches.tfm), $Agent)" + $runTitle = "$($matches.lib) ($($matches.tfm)$x86RunTitleSuffix, $Agent)" } } } if (!$runTitle) { $unknownCounter += 1; - $runTitle = "unknown$unknownCounter ($Agent)"; + $runTitle = "unknown$unknownCounter ($Agent$x86RunTitleSuffix)"; } Write-Host "##vso[results.publish type=VSTest;runTitle=$runTitle;publishRunAttachments=true;resultFiles=$_;failTaskOnFailedTests=true;testRunSystem=VSTS - PTR;]" diff --git a/tools/Install-DotNetSdk.ps1 b/tools/Install-DotNetSdk.ps1 index 7dcddc78..76a397f0 100644 --- a/tools/Install-DotNetSdk.ps1 +++ b/tools/Install-DotNetSdk.ps1 @@ -15,20 +15,28 @@ When using 'repo', environment variables are set to cause the locally installed dotnet SDK to be used. Per-repo can lead to file locking issues when dotnet.exe is left running as a build server and can be mitigated by running `dotnet build-server shutdown`. Per-machine requires elevation and will download and install all SDKs and runtimes to machine-wide locations so all applications can find it. +.PARAMETER IncludeX86 + Installs a x86 SDK and runtimes in addition to the x64 ones. Only supported on Windows. Ignored on others. #> [CmdletBinding(SupportsShouldProcess=$true,ConfirmImpact='Medium')] Param ( [ValidateSet('repo','user','machine')] - [string]$InstallLocality='user' + [string]$InstallLocality='user', + [switch]$IncludeX86 ) $DotNetInstallScriptRoot = "$PSScriptRoot/../obj/tools" if (!(Test-Path $DotNetInstallScriptRoot)) { New-Item -ItemType Directory -Path $DotNetInstallScriptRoot -WhatIf:$false | Out-Null } $DotNetInstallScriptRoot = Resolve-Path $DotNetInstallScriptRoot -# Look up actual required .NET Core SDK version from global.json +# Look up actual required .NET SDK version from global.json $sdkVersion = & "$PSScriptRoot/../azure-pipelines/variables/DotNetSdkVersion.ps1" +If ($IncludeX86 -and ($IsMacOS -or $IsLinux)) { + Write-Verbose "Ignoring -IncludeX86 switch because 32-bit runtimes are only supported on Windows." + $IncludeX86 = $false +} + $arch = [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture if (!$arch) { # Windows Powershell leaves this blank $arch = 'x64' @@ -72,7 +80,7 @@ Function Get-FileFromWeb([Uri]$Uri, $OutDir) { $OutFile = Join-Path $OutDir $Uri.Segments[-1] if (!(Test-Path $OutFile)) { Write-Verbose "Downloading $Uri..." - if (!(Test-Path $OutDir)) { mkdir $OutDir } + if (!(Test-Path $OutDir)) { New-Item -ItemType Directory -Path $OutDir } try { (New-Object System.Net.WebClient).DownloadFile($Uri, $OutFile) } finally { @@ -83,7 +91,7 @@ Function Get-FileFromWeb([Uri]$Uri, $OutDir) { $OutFile } -Function Get-InstallerExe($Version, [switch]$Runtime) { +Function Get-InstallerExe($Version, $Architecture, [switch]$Runtime) { $sdkOrRuntime = 'Sdk' if ($Runtime) { $sdkOrRuntime = 'Runtime' } @@ -116,7 +124,7 @@ Function Get-InstallerExe($Version, [switch]$Runtime) { if ($filesElement) { foreach ($file in $filesElement) { - if ($file.rid -eq "win-$arch") { + if ($file.rid -eq "win-$Architecture") { $url = $file.url Break } @@ -135,22 +143,20 @@ Function Get-InstallerExe($Version, [switch]$Runtime) { } } -Function Install-DotNet($Version, [switch]$Runtime) { +Function Install-DotNet($Version, $Architecture, [switch]$Runtime) { if ($Runtime) { $sdkSubstring = '' } else { $sdkSubstring = 'SDK ' } Write-Host "Downloading .NET Core $sdkSubstring$Version..." - $Installer = Get-InstallerExe -Version $Version -Runtime:$Runtime + $Installer = Get-InstallerExe -Version $Version -Architecture $Architecture -Runtime:$Runtime Write-Host "Installing .NET Core $sdkSubstring$Version..." cmd /c start /wait $Installer /install /passive /norestart if ($LASTEXITCODE -eq 3010) { Write-Verbose "Restart required" } elseif ($LASTEXITCODE -ne 0) { - throw "Failure to install .NET Core SDK" + throw "Failure to install .NET SDK" } } -$switches = @( - '-Architecture',$arch -) +$switches = @() $envVars = @{ # For locally installed dotnet, skip first time experience which takes a long time 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' = 'true'; @@ -161,15 +167,25 @@ if ($InstallLocality -eq 'machine') { $DotNetInstallDir = '/usr/share/dotnet' } else { $restartRequired = $false - if ($PSCmdlet.ShouldProcess(".NET Core SDK $sdkVersion", "Install")) { - Install-DotNet -Version $sdkVersion + if ($PSCmdlet.ShouldProcess(".NET SDK $sdkVersion", "Install")) { + Install-DotNet -Version $sdkVersion -Architecture $arch $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + + if ($IncludeX86) { + Install-DotNet -Version $sdkVersion -Architecture x86 + $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + } } $runtimeVersions | Get-Unique |% { if ($PSCmdlet.ShouldProcess(".NET Core runtime $_", "Install")) { - Install-DotNet -Version $_ -Runtime + Install-DotNet -Version $_ -Architecture $arch -Runtime $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + + if ($IncludeX86) { + Install-DotNet -Version $_ -Architecture x86 -Runtime + $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + } } } @@ -182,20 +198,34 @@ if ($InstallLocality -eq 'machine') { } } elseif ($InstallLocality -eq 'repo') { $DotNetInstallDir = "$DotNetInstallScriptRoot/.dotnet" + $DotNetX86InstallDir = "$DotNetInstallScriptRoot/x86/.dotnet" } elseif ($env:AGENT_TOOLSDIRECTORY) { $DotNetInstallDir = "$env:AGENT_TOOLSDIRECTORY/dotnet" + $DotNetX86InstallDir = "$env:AGENT_TOOLSDIRECTORY/x86/dotnet" } else { $DotNetInstallDir = Join-Path $HOME .dotnet } -Write-Host "Installing .NET Core SDK and runtimes to $DotNetInstallDir" -ForegroundColor Blue - if ($DotNetInstallDir) { - $switches += '-InstallDir',"`"$DotNetInstallDir`"" + if (!(Test-Path $DotNetInstallDir)) { New-Item -ItemType Directory -Path $DotNetInstallDir } + $DotNetInstallDir = Resolve-Path $DotNetInstallDir + Write-Host "Installing .NET SDK and runtimes to $DotNetInstallDir" -ForegroundColor Blue $envVars['DOTNET_MULTILEVEL_LOOKUP'] = '0' $envVars['DOTNET_ROOT'] = $DotNetInstallDir } +if ($IncludeX86) { + if ($DotNetX86InstallDir) { + if (!(Test-Path $DotNetX86InstallDir)) { New-Item -ItemType Directory -Path $DotNetX86InstallDir } + $DotNetX86InstallDir = Resolve-Path $DotNetX86InstallDir + Write-Host "Installing x86 .NET SDK and runtimes to $DotNetX86InstallDir" -ForegroundColor Blue + } else { + # Only machine-wide or repo-wide installations can handle two unique dotnet.exe architectures. + Write-Error "The installation location or OS isn't supported for x86 installation. Try a different -InstallLocality value." + return 1 + } +} + if ($IsMacOS -or $IsLinux) { $DownloadUri = "https://raw.githubusercontent.com/dotnet/install-scripts/781752509a890ca7520f1182e8bae71f9a53d754/src/dotnet-install.sh" $DotNetInstallScriptPath = "$DotNetInstallScriptRoot/dotnet-install.sh" @@ -219,47 +249,89 @@ $DotNetInstallScriptPathExpression = "& '$DotNetInstallScriptPathExpression'" $anythingInstalled = $false $global:LASTEXITCODE = 0 -if ($PSCmdlet.ShouldProcess(".NET Core SDK $sdkVersion", "Install")) { +if ($PSCmdlet.ShouldProcess(".NET SDK $sdkVersion", "Install")) { $anythingInstalled = $true - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion $switches" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion -Architecture $arch -InstallDir $DotNetInstallDir $switches" if ($LASTEXITCODE -ne 0) { Write-Error ".NET SDK installation failure: $LASTEXITCODE" exit $LASTEXITCODE } } else { - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion $switches -DryRun" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion -Architecture $arch -InstallDir $DotNetInstallDir $switches -DryRun" +} + +if ($IncludeX86) { + if ($PSCmdlet.ShouldProcess(".NET x86 SDK $sdkVersion", "Install")) { + $anythingInstalled = $true + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion -Architecture x86 -InstallDir $DotNetX86InstallDir $switches" + + if ($LASTEXITCODE -ne 0) { + Write-Error ".NET x86 SDK installation failure: $LASTEXITCODE" + exit $LASTEXITCODE + } + } else { + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion -Architecture x86 -InstallDir $DotNetX86InstallDir $switches -DryRun" + } } $dotnetRuntimeSwitches = $switches + '-Runtime','dotnet' $runtimeVersions | Sort-Object -Unique |% { - if ($PSCmdlet.ShouldProcess(".NET Core runtime $_", "Install")) { + if ($PSCmdlet.ShouldProcess(".NET Core $Arch runtime $_", "Install")) { $anythingInstalled = $true - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ $dotnetRuntimeSwitches" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture $arch -InstallDir $DotNetInstallDir $dotnetRuntimeSwitches" if ($LASTEXITCODE -ne 0) { Write-Error ".NET SDK installation failure: $LASTEXITCODE" exit $LASTEXITCODE } } else { - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ $dotnetRuntimeSwitches -DryRun" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture $arch -InstallDir $DotNetInstallDir $dotnetRuntimeSwitches -DryRun" + } + + if ($IncludeX86) { + if ($PSCmdlet.ShouldProcess(".NET Core x86 runtime $_", "Install")) { + $anythingInstalled = $true + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture x86 -InstallDir $DotNetX86InstallDir $dotnetRuntimeSwitches" + + if ($LASTEXITCODE -ne 0) { + Write-Error ".NET SDK installation failure: $LASTEXITCODE" + exit $LASTEXITCODE + } + } else { + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture x86 -InstallDir $DotNetX86InstallDir $dotnetRuntimeSwitches -DryRun" + } } } $windowsDesktopRuntimeSwitches = $switches + '-Runtime','windowsdesktop' $windowsDesktopRuntimeVersions | Sort-Object -Unique |% { - if ($PSCmdlet.ShouldProcess(".NET Core WindowsDesktop runtime $_", "Install")) { + if ($PSCmdlet.ShouldProcess(".NET Core WindowsDesktop $arch runtime $_", "Install")) { $anythingInstalled = $true - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ $windowsDesktopRuntimeSwitches" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture $arch -InstallDir $DotNetInstallDir $windowsDesktopRuntimeSwitches" if ($LASTEXITCODE -ne 0) { Write-Error ".NET SDK installation failure: $LASTEXITCODE" exit $LASTEXITCODE } } else { - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ $windowsDesktopRuntimeSwitches -DryRun" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture $arch -InstallDir $DotNetInstallDir $windowsDesktopRuntimeSwitches -DryRun" + } + + if ($IncludeX86) { + if ($PSCmdlet.ShouldProcess(".NET Core WindowsDesktop x86 runtime $_", "Install")) { + $anythingInstalled = $true + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture x86 -InstallDir $DotNetX86InstallDir $windowsDesktopRuntimeSwitches" + + if ($LASTEXITCODE -ne 0) { + Write-Error ".NET SDK installation failure: $LASTEXITCODE" + exit $LASTEXITCODE + } + } else { + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture x86 -InstallDir $DotNetX86InstallDir $windowsDesktopRuntimeSwitches -DryRun" + } } } From 9805678248d0fbda63f97982f37e35de71890d2f Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 12 May 2022 08:41:05 -0600 Subject: [PATCH 57/99] Fix NUGET_PACKAGES path in pipelines Be default the NUGET_PACKAGES path has a trailing slash. In our override, we should do this as well or the msbuild property that gets generated is missing the slash, which can break some builds. --- .github/workflows/build.yml | 2 +- azure-pipelines.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18ae3e28..4db43f53 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BUILDCONFIGURATION: Release codecov_token: 4dc9e7e2-6b01-4932-a180-847b52b43d35 # Get a new one from https://codecov.io/ - NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages + NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages/ jobs: build: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b7d86b45..d98ed107 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,7 +24,7 @@ variables: BuildConfiguration: Release codecov_token: 4dc9e7e2-6b01-4932-a180-847b52b43d35 # Get a new one from https://codecov.io/ ci_feed: https://pkgs.dev.azure.com/andrewarnott/_packaging/CI/nuget/v3/index.json # Azure Artifacts feed URL - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages + NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ jobs: - template: azure-pipelines/build.yml From da08e901c94c416cbbdfa980020ced9ed9612851 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 12 May 2022 16:08:56 -0600 Subject: [PATCH 58/99] Add ability to split stage and publish artifact steps --- azure-pipelines/artifacts/_all.ps1 | 10 +++---- azure-pipelines/artifacts/_pipelines.ps1 | 35 ++++++++++++++++++++---- azure-pipelines/artifacts/_stage_all.ps1 | 8 +++++- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/azure-pipelines/artifacts/_all.ps1 b/azure-pipelines/artifacts/_all.ps1 index afe42be3..c9182a45 100755 --- a/azure-pipelines/artifacts/_all.ps1 +++ b/azure-pipelines/artifacts/_all.ps1 @@ -12,7 +12,7 @@ Value = an array of paths (absolute or relative to the BaseDirectory) to files to include in the artifact. FileInfo objects are also allowed. .PARAMETER Force - Executes artifact scripts even if they have already been uploaded. + Executes artifact scripts even if they have already been staged. #> param ( @@ -28,14 +28,14 @@ Function EnsureTrailingSlash($path) { $path.Replace('\', [IO.Path]::DirectorySeparatorChar) } -Function Test-ArtifactUploaded($artifactName) { - $varName = "ARTIFACTUPLOADED_$($artifactName.ToUpper())" +Function Test-ArtifactStaged($artifactName) { + $varName = "ARTIFACTSTAGED_$($artifactName.ToUpper())" Test-Path "env:$varName" } Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" -Recurse | % { $ArtifactName = $_.BaseName - if ($Force -or !(Test-ArtifactUploaded($ArtifactName + $ArtifactNameSuffix))) { + if ($Force -or !(Test-ArtifactStaged($ArtifactName + $ArtifactNameSuffix))) { $totalFileCount = 0 $fileGroups = & $_ if ($fileGroups) { @@ -65,6 +65,6 @@ Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" -Recurse | % { Write-Warning "No files found for the `"$ArtifactName`" artifact." } } else { - Write-Host "Skipping $ArtifactName because it has already been uploaded." -ForegroundColor DarkGray + Write-Host "Skipping $ArtifactName because it has already been staged." -ForegroundColor DarkGray } } diff --git a/azure-pipelines/artifacts/_pipelines.ps1 b/azure-pipelines/artifacts/_pipelines.ps1 index 73a3af0a..a62d2675 100644 --- a/azure-pipelines/artifacts/_pipelines.ps1 +++ b/azure-pipelines/artifacts/_pipelines.ps1 @@ -2,14 +2,39 @@ # into commands that instruct Azure Pipelines to actually collect those artifacts. param ( - [string]$ArtifactNameSuffix + [string]$ArtifactNameSuffix, + [switch]$StageOnly ) -& "$PSScriptRoot/_stage_all.ps1" -ArtifactNameSuffix $ArtifactNameSuffix |% { - Write-Host "##vso[artifact.upload containerfolder=$($_.Name);artifactname=$($_.Name);]$($_.Path)" +Function Set-PipelineVariable($name, $value) { + if ((Test-Path "Env:\$name") -and (Get-Item "Env:\$name").Value -eq $value) { + return # already set + } + + #New-Item -Path "Env:\$name".ToUpper() -Value $value -Force | Out-Null + Write-Host "##vso[task.setvariable variable=$name]$value" +} + +Function Test-ArtifactUploaded($artifactName) { + $varName = "ARTIFACTUPLOADED_$($artifactName.ToUpper())" + Test-Path "env:$varName" +} +& "$PSScriptRoot/_stage_all.ps1" -ArtifactNameSuffix $ArtifactNameSuffix |% { # Set a variable which will out-live this script so that a subsequent attempt to collect and upload artifacts # will skip this one from a check in the _all.ps1 script. - $varName = "ARTIFACTUPLOADED_$($_.Name.ToUpper())" - Write-Host "##vso[task.setvariable variable=$varName]true" + Set-PipelineVariable "ARTIFACTSTAGED_$($_.Name.ToUpper())" 'true' + Write-Host "Staged artifact $($_.Name) to $($_.Path)" + + if (!$StageOnly) { + if (Test-ArtifactUploaded $_.Name) { + Write-Host "Skipping $($_.Name) because it has already been uploaded." -ForegroundColor DarkGray + } else { + Write-Host "##vso[artifact.upload containerfolder=$($_.Name);artifactname=$($_.Name);]$($_.Path)" + + # Set a variable which will out-live this script so that a subsequent attempt to collect and upload artifacts + # will skip this one from a check in the _all.ps1 script. + Set-PipelineVariable "ARTIFACTUPLOADED_$($_.Name.ToUpper())" 'true' + } + } } diff --git a/azure-pipelines/artifacts/_stage_all.ps1 b/azure-pipelines/artifacts/_stage_all.ps1 index e4954c13..b7166b4e 100644 --- a/azure-pipelines/artifacts/_stage_all.ps1 +++ b/azure-pipelines/artifacts/_stage_all.ps1 @@ -42,7 +42,13 @@ $Artifacts |% { } } -$Artifacts |% { "$($_.ArtifactName)$ArtifactNameSuffix" } | Get-Unique |% { +$ArtifactNames = $Artifacts |% { "$($_.ArtifactName)$ArtifactNameSuffix" } +$ArtifactNames += Get-ChildItem env:ARTIFACTSTAGED_* |% { + # Return from ALLCAPS to the actual capitalization used for the artifact. + $artifactNameAllCaps = "$($_.Name.Substring('ARTIFACTSTAGED_'.Length))" + (Get-ChildItem $ArtifactStagingFolder\$artifactNameAllCaps* -Filter $artifactNameAllCaps).Name +} +$ArtifactNames | Get-Unique |% { $artifact = New-Object -TypeName PSObject Add-Member -InputObject $artifact -MemberType NoteProperty -Name Name -Value $_ Add-Member -InputObject $artifact -MemberType NoteProperty -Name Path -Value (Join-Path $ArtifactStagingFolder $_) From c268a9e8d99bfed18778aaecea828720522c6dea Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 18 May 2022 16:18:24 -0600 Subject: [PATCH 59/99] Fix several Install-DotNetSdk issues * Stop popping Windows Explorer dialogs * Stop installing the same .NET runtime versions multiple times. * Install WindowsDesktop too. --- tools/Install-DotNetSdk.ps1 | 57 +++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/tools/Install-DotNetSdk.ps1 b/tools/Install-DotNetSdk.ps1 index 76a397f0..2bac3b9b 100644 --- a/tools/Install-DotNetSdk.ps1 +++ b/tools/Install-DotNetSdk.ps1 @@ -80,7 +80,7 @@ Function Get-FileFromWeb([Uri]$Uri, $OutDir) { $OutFile = Join-Path $OutDir $Uri.Segments[-1] if (!(Test-Path $OutFile)) { Write-Verbose "Downloading $Uri..." - if (!(Test-Path $OutDir)) { New-Item -ItemType Directory -Path $OutDir } + if (!(Test-Path $OutDir)) { New-Item -ItemType Directory -Path $OutDir | Out-Null } try { (New-Object System.Net.WebClient).DownloadFile($Uri, $OutFile) } finally { @@ -91,35 +91,31 @@ Function Get-FileFromWeb([Uri]$Uri, $OutDir) { $OutFile } -Function Get-InstallerExe($Version, $Architecture, [switch]$Runtime) { - $sdkOrRuntime = 'Sdk' - if ($Runtime) { $sdkOrRuntime = 'Runtime' } - +Function Get-InstallerExe( + $Version, + $Architecture, + [ValidateSet('Sdk','Runtime','WindowsDesktop')] + [string]$sku +) { # Get the latest/actual version for the specified one $TypedVersion = [Version]$Version if ($TypedVersion.Build -eq -1) { - $versionInfo = -Split (Invoke-WebRequest -Uri "https://dotnetcli.blob.core.windows.net/dotnet/$sdkOrRuntime/$Version/latest.version" -UseBasicParsing) + $versionInfo = -Split (Invoke-WebRequest -Uri "https://dotnetcli.blob.core.windows.net/dotnet/$sku/$Version/latest.version" -UseBasicParsing) $Version = $versionInfo[-1] } $majorMinor = "$($TypedVersion.Major).$($TypedVersion.Minor)" $ReleasesFile = Join-Path $DotNetInstallScriptRoot "$majorMinor\releases.json" if (!(Test-Path $ReleasesFile)) { - Get-FileFromWeb -Uri "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/$majorMinor/releases.json" -OutDir (Split-Path $ReleasesFile) + Get-FileFromWeb -Uri "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/$majorMinor/releases.json" -OutDir (Split-Path $ReleasesFile) | Out-Null } $releases = Get-Content $ReleasesFile | ConvertFrom-Json $url = $null foreach ($release in $releases.releases) { $filesElement = $null - if ($Runtime) { - if ($release.runtime.version -eq $Version) { - $filesElement = $release.runtime.files - } - } else { - if ($release.sdk.version -eq $Version) { - $filesElement = $release.sdk.files - } + if ($release.$sku.version -eq $Version) { + $filesElement = $release.$sku.files } if ($filesElement) { @@ -139,15 +135,14 @@ Function Get-InstallerExe($Version, $Architecture, [switch]$Runtime) { if ($url) { Get-FileFromWeb -Uri $url -OutDir $DotNetInstallScriptRoot } else { - Write-Error "Unable to find release of $sdkOrRuntime v$Version" + Write-Error "Unable to find release of $sku v$Version" } } -Function Install-DotNet($Version, $Architecture, [switch]$Runtime) { - if ($Runtime) { $sdkSubstring = '' } else { $sdkSubstring = 'SDK ' } - Write-Host "Downloading .NET Core $sdkSubstring$Version..." - $Installer = Get-InstallerExe -Version $Version -Architecture $Architecture -Runtime:$Runtime - Write-Host "Installing .NET Core $sdkSubstring$Version..." +Function Install-DotNet($Version, $Architecture, [ValidateSet('Sdk','Runtime','WindowsDesktop')][string]$sku = 'Sdk') { + Write-Host "Downloading .NET Core $sku $Version..." + $Installer = Get-InstallerExe -Version $Version -Architecture $Architecture -sku $sku + Write-Host "Installing .NET Core $sku $Version..." cmd /c start /wait $Installer /install /passive /norestart if ($LASTEXITCODE -eq 3010) { Write-Verbose "Restart required" @@ -177,13 +172,25 @@ if ($InstallLocality -eq 'machine') { } } - $runtimeVersions | Get-Unique |% { - if ($PSCmdlet.ShouldProcess(".NET Core runtime $_", "Install")) { - Install-DotNet -Version $_ -Architecture $arch -Runtime + $runtimeVersions | Sort-Object | Get-Unique |% { + if ($PSCmdlet.ShouldProcess(".NET runtime $_", "Install")) { + Install-DotNet -Version $_ -sku Runtime -Architecture $arch + $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + + if ($IncludeX86) { + Install-DotNet -Version $_ -sku Runtime -Architecture x86 + $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + } + } + } + + $windowsDesktopRuntimeVersions | Sort-Object | Get-Unique |% { + if ($PSCmdlet.ShouldProcess(".NET Windows Desktop $_", "Install")) { + Install-DotNet -Version $_ -sku WindowsDesktop -Architecture $arch $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) if ($IncludeX86) { - Install-DotNet -Version $_ -Architecture x86 -Runtime + Install-DotNet -Version $_ -sku WindowsDesktop -Architecture x86 $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) } } From 7bf4e22b5ab48ca5b0b3992804f685d9b857dad1 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 20 May 2022 14:06:55 -0600 Subject: [PATCH 60/99] Fix Apply-Template merge check --- Apply-Template.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apply-Template.ps1 b/Apply-Template.ps1 index d659670b..12bfaa12 100644 --- a/Apply-Template.ps1 +++ b/Apply-Template.ps1 @@ -20,7 +20,7 @@ try { # Look for our own initial commit in the target repo's history. # If it's there, they've already switched to using git merge to freshen up. # Using Apply-Template would just complicate future merges, so block it. - git log 05f49ce799c1f9cc696d53eea89699d80f59f833 ^HEAD | Out-Null + git merge-base --is-ancestor 05f49ce799c1f9cc696d53eea89699d80f59f833 HEAD | Out-Null if ($LASTEXITCODE -eq 0) { Write-Error 'The target repo already has Library.Template history merged into it. Use `git merge` instead of this script to freshen your repo. See the README.md file for details.' exit 1 From 5abdd709c599b95cb6f584eef289fe665031116a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jun 2022 08:24:16 -0600 Subject: [PATCH 61/99] Bump Microsoft.NET.Test.Sdk from 17.1.0 to 17.2.0 (#159) Bumps [Microsoft.NET.Test.Sdk](https://github.com/microsoft/vstest) from 17.1.0 to 17.2.0. - [Release notes](https://github.com/microsoft/vstest/releases) - [Commits](https://github.com/microsoft/vstest/compare/v17.1.0...v17.2.0) --- updated-dependencies: - dependency-name: Microsoft.NET.Test.Sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test/Library.Tests/Library.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Library.Tests/Library.Tests.csproj b/test/Library.Tests/Library.Tests.csproj index 20108645..7dd444b6 100644 --- a/test/Library.Tests/Library.Tests.csproj +++ b/test/Library.Tests/Library.Tests.csproj @@ -11,7 +11,7 @@ - + From 311e766eea33c9ea74f6b6b444fa2cb6463722c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jun 2022 14:24:39 +0000 Subject: [PATCH 62/99] Bump Nerdbank.GitVersioning from 3.5.103 to 3.5.107 (#157) --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 2e13d76f..a2da14f6 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -37,7 +37,7 @@ - + From 6a1593c66812b348812be702b41320290839ce0c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jun 2022 14:24:49 +0000 Subject: [PATCH 63/99] Bump StyleCop.Analyzers.Unstable from 1.2.0.406 to 1.2.0.435 (#156) --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index a2da14f6..22c1b94d 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -40,7 +40,7 @@ - + From a91a2f1e2331902ab7515f1d317cf105892b9cb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jun 2022 14:31:58 +0000 Subject: [PATCH 64/99] Bump xunit.runner.visualstudio from 2.4.3 to 2.4.5 (#158) --- test/Library.Tests/Library.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Library.Tests/Library.Tests.csproj b/test/Library.Tests/Library.Tests.csproj index 7dd444b6..402468a8 100644 --- a/test/Library.Tests/Library.Tests.csproj +++ b/test/Library.Tests/Library.Tests.csproj @@ -12,7 +12,7 @@ - + From 8f2ed5eaf2436e9e49e741650308c0b45b4a53df Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 7 Jun 2022 15:50:23 -0600 Subject: [PATCH 65/99] Update SDK to 6.0.300 --- .devcontainer/Dockerfile | 2 +- global.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 234db6d1..b13ca0ac 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:6.0.200-focal +FROM mcr.microsoft.com/dotnet/sdk:6.0.300-focal # Installing mono makes `dotnet test` work without errors even for net472. # But installing it takes a long time, so it's excluded by default. diff --git a/global.json b/global.json index 6bfbd177..954a92e7 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "6.0.200", + "version": "6.0.300", "rollForward": "patch", "allowPrerelease": false } From 8c6b68ecdd6cb98a02bc89e762d4cf8851e0148b Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 8 Jun 2022 08:48:38 -0600 Subject: [PATCH 66/99] Add adoption caution about github issues being closed --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 0f9715f3..8cab6b7a 100644 --- a/README.md +++ b/README.md @@ -79,5 +79,9 @@ We create the merge commit with these commands: 1. Run `git commit-tree -p HEAD -p A B -m "Merged latest Library.Template"`, where `A` is the output from `git rev-parse HEAD` that you recorded earlier, and `B` is the output from your prior `git write-tree` command. 1. Run `git merge X` where `X` is the output of the `git commit-tree` command. +**CAUTION**: when merging this for the first time, a github-hosted repo may close issues in your repo with the same number as issues that this repo closed in git commit messages. +Verify after completing your PR by visiting your github closed issues, sorted by recently updated, and reactivate any that were inadvertently closed by this merge. +This shouldn't be a recurring issue because going forward, we will avoid referencing github issues with simple `#123` syntax in this repo's history. + Congratulations. You're all done. Next time you want to sync to latest from Library.Template, you can the simple `git merge` steps given at the start of this section. From f678dd36be37f69272b6346b4ba3acb7472ec418 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 8 Jun 2022 12:12:42 -0600 Subject: [PATCH 67/99] Install the nuget cred provider for .NET 6 and .NET Framework Recently we've only been getting the .NET Core 3.1 plugin, which leaves .NET Framework-based nuget tools un-authenticated. --- tools/Install-NuGetCredProvider.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/Install-NuGetCredProvider.ps1 b/tools/Install-NuGetCredProvider.ps1 index 6d310034..496049a2 100755 --- a/tools/Install-NuGetCredProvider.ps1 +++ b/tools/Install-NuGetCredProvider.ps1 @@ -33,7 +33,7 @@ if ($IsMacOS -or $IsLinux) { $installerScript = Join-Path $toolsPath $installerScript -if (!(Test-Path $installerScript)) { +if (!(Test-Path $installerScript) -or $Force) { Invoke-WebRequest $sourceUrl -OutFile $installerScript } @@ -43,7 +43,7 @@ if ($IsMacOS -or $IsLinux) { chmod u+x $installerScript } -& $installerScript -Force:$Force +& $installerScript -Force:$Force -AddNetfx -InstallNet6 if ($AccessToken) { $endpoints = @() From d8f567ef793329d7ff8cc09cf43922d2d00316c9 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 8 Jun 2022 08:58:02 -0600 Subject: [PATCH 68/99] Avoid truncated code coverage data Apply workaround for microsoft/vstest issue 3724. --- azure-pipelines/artifacts/coverageResults.ps1 | 7 +-- azure-pipelines/dotnet-test-cloud.ps1 | 3 +- azure-pipelines/publish-codecoverage.yml | 7 +-- azure-pipelines/test.runsettings | 44 +++++++++++++++++++ test/Directory.Build.targets | 7 --- test/Library.Tests/Library.Tests.csproj | 1 - 6 files changed, 54 insertions(+), 15 deletions(-) create mode 100644 azure-pipelines/test.runsettings diff --git a/azure-pipelines/artifacts/coverageResults.ps1 b/azure-pipelines/artifacts/coverageResults.ps1 index 8fdb3f72..b3a08891 100644 --- a/azure-pipelines/artifacts/coverageResults.ps1 +++ b/azure-pipelines/artifacts/coverageResults.ps1 @@ -1,10 +1,11 @@ $RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..") +$coverageFiles = @(Get-ChildItem "$RepoRoot/test/*.cobertura.xml" -Recurse) + # Prepare code coverage reports for merging on another machine if ($env:SYSTEM_DEFAULTWORKINGDIRECTORY) { Write-Host "Substituting $env:SYSTEM_DEFAULTWORKINGDIRECTORY with `"{reporoot}`"" - $reports = Get-ChildItem "$RepoRoot/bin/coverage.*cobertura.xml" -Recurse - $reports |% { + $coverageFiles |% { $content = Get-Content -Path $_ |% { $_ -Replace [regex]::Escape($env:SYSTEM_DEFAULTWORKINGDIRECTORY), "{reporoot}" } Set-Content -Path $_ -Value $content -Encoding UTF8 } @@ -16,7 +17,7 @@ if (!((Test-Path $RepoRoot\bin) -and (Test-Path $RepoRoot\obj))) { return } @{ $RepoRoot = ( - @(Get-ChildItem "$RepoRoot\bin\coverage.*cobertura.xml" -Recurse) + + $coverageFiles + (Get-ChildItem "$RepoRoot\obj\*.cs" -Recurse) ); } diff --git a/azure-pipelines/dotnet-test-cloud.ps1 b/azure-pipelines/dotnet-test-cloud.ps1 index 43cdc3e0..24bf812a 100755 --- a/azure-pipelines/dotnet-test-cloud.ps1 +++ b/azure-pipelines/dotnet-test-cloud.ps1 @@ -48,7 +48,8 @@ if ($x86) { --no-build ` -c $Configuration ` --filter "TestCategory!=FailsInCloudTest" ` - -p:CollectCoverage=true ` + --collect "Code Coverage;Format=cobertura" ` + --settings "$PSScriptRoot/test.runsettings" ` --blame-hang-timeout 60s ` --blame-crash ` -bl:"$ArtifactStagingFolder/build_logs/test.binlog" ` diff --git a/azure-pipelines/publish-codecoverage.yml b/azure-pipelines/publish-codecoverage.yml index 423e12fc..4bc1b3bb 100644 --- a/azure-pipelines/publish-codecoverage.yml +++ b/azure-pipelines/publish-codecoverage.yml @@ -16,14 +16,15 @@ steps: continueOnError: true condition: ${{ parameters.includeMacOS }} - powershell: | - dotnet tool install --tool-path obj dotnet-reportgenerator-globaltool --version 4.8.5 --configfile azure-pipelines/justnugetorg.nuget.config + dotnet tool install --tool-path obj dotnet-reportgenerator-globaltool --version 5.1.9 --configfile azure-pipelines/justnugetorg.nuget.config Copy-Item -Recurse $(Pipeline.Workspace)/coverageResults-Windows/obj/* $(System.DefaultWorkingDirectory)/obj Write-Host 'Substituting {reporoot} with $(System.DefaultWorkingDirectory)' - $reports = Get-ChildItem -Recurse '$(Pipeline.Workspace)/coverage.*cobertura.xml' + $reports = Get-ChildItem -Recurse '$(Pipeline.Workspace)/*.cobertura.xml' $reports |% { $content = Get-Content -Path $_ |% { $_.Replace('{reporoot}', '$(System.DefaultWorkingDirectory)') } Set-Content -Path $_ -Value $content -Encoding UTF8 } + $Inputs = [string]::join(';', ($reports |% { Resolve-Path -relative $_ })) obj/reportgenerator -reports:"$Inputs" -targetdir:coveragereport -reporttypes:Cobertura displayName: โš™ Merge coverage @@ -31,5 +32,5 @@ steps: displayName: ๐Ÿ“ข Publish code coverage results to Azure DevOps inputs: codeCoverageTool: cobertura - summaryFileLocation: 'coveragereport/Cobertura.xml' + summaryFileLocation: coveragereport/Cobertura.xml failIfCoverageEmpty: true diff --git a/azure-pipelines/test.runsettings b/azure-pipelines/test.runsettings new file mode 100644 index 00000000..c69022fc --- /dev/null +++ b/azure-pipelines/test.runsettings @@ -0,0 +1,44 @@ + + + + + + + + + \.dll$ + \.exe$ + + + xunit\..* + + + + + ^System\.Diagnostics\.DebuggerHiddenAttribute$ + ^System\.Diagnostics\.DebuggerNonUserCodeAttribute$ + ^System\.CodeDom\.Compiler\.GeneratedCodeAttribute$ + ^System\.Diagnostics\.CodeAnalysis\.ExcludeFromCodeCoverageAttribute$ + + + + + True + + True + + True + + False + + True + + True + + True + + + + + + diff --git a/test/Directory.Build.targets b/test/Directory.Build.targets index 2faab375..e7edee55 100644 --- a/test/Directory.Build.targets +++ b/test/Directory.Build.targets @@ -1,10 +1,3 @@ - - cobertura - [xunit.*]* - - $(OutputPath)/ - - diff --git a/test/Library.Tests/Library.Tests.csproj b/test/Library.Tests/Library.Tests.csproj index 402468a8..92062dc7 100644 --- a/test/Library.Tests/Library.Tests.csproj +++ b/test/Library.Tests/Library.Tests.csproj @@ -10,7 +10,6 @@ - From f064b9a150eb529716d393fc0f1f60b55a301eb4 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 8 Jun 2022 16:22:01 -0600 Subject: [PATCH 69/99] Normalize slashes in coverage --- azure-pipelines/publish-codecoverage.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines/publish-codecoverage.yml b/azure-pipelines/publish-codecoverage.yml index 4bc1b3bb..23e88a6e 100644 --- a/azure-pipelines/publish-codecoverage.yml +++ b/azure-pipelines/publish-codecoverage.yml @@ -21,7 +21,8 @@ steps: Write-Host 'Substituting {reporoot} with $(System.DefaultWorkingDirectory)' $reports = Get-ChildItem -Recurse '$(Pipeline.Workspace)/*.cobertura.xml' $reports |% { - $content = Get-Content -Path $_ |% { $_.Replace('{reporoot}', '$(System.DefaultWorkingDirectory)') } + # In addition to replacing {reporoot}, we also normalize on one kind of slash so that the report aggregates data for a file whether data was collected on Windows or not. + $content = Get-Content -Path $_ |% { [Regex]::Replace($_, '{reporoot}([^"]+)', { '$(System.DefaultWorkingDirectory)' + $args[0].groups[1].value.replace([IO.Path]::AltDirectorySeparatorChar, [IO.Path]::DirectorySeparatorChar) }) } Set-Content -Path $_ -Value $content -Encoding UTF8 } From e903112e9ebff59ebad913bf6c89b14f97f707b1 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 14 Jun 2022 13:20:10 -0600 Subject: [PATCH 70/99] Include only unique coverage reports in artifact The test runner is creating *two* cobertura.xml files for each test run. Besides doubling the size of the coverageResults artifact, this makes the `publish-codecoverage.yml` pipeline step do much more work which can take *minutes* instead of seconds. --- azure-pipelines/artifacts/coverageResults.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/artifacts/coverageResults.ps1 b/azure-pipelines/artifacts/coverageResults.ps1 index b3a08891..280ff9ae 100644 --- a/azure-pipelines/artifacts/coverageResults.ps1 +++ b/azure-pipelines/artifacts/coverageResults.ps1 @@ -1,6 +1,6 @@ $RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..") -$coverageFiles = @(Get-ChildItem "$RepoRoot/test/*.cobertura.xml" -Recurse) +$coverageFiles = @(Get-ChildItem "$RepoRoot/test/*.cobertura.xml" -Recurse | Where {$_.FullName -notlike "*/In/*" -and $_.FullName -notlike "*\In\*" }) # Prepare code coverage reports for merging on another machine if ($env:SYSTEM_DEFAULTWORKINGDIRECTORY) { From c881dcad9590e33952a956950d6a2e4c4249c0ef Mon Sep 17 00:00:00 2001 From: Steve Bush Date: Wed, 15 Jun 2022 09:02:52 -0700 Subject: [PATCH 71/99] CodeCov binary uploader Replaces the bash CodeCov uploader with the latest binary version. A PowerShell script downloads the latest codecov binary. Another PowerShell script collects and uploads code coverage artifact files to CodeCov.io. You can pass flags and a coverage report name as parameters. --- .github/workflows/build.yml | 5 ++-- azure-pipelines/Get-CodeCovTool.ps1 | 38 ++++++++++++++++++++++++++ azure-pipelines/dotnet.yml | 5 +++- azure-pipelines/publish-CodeCov.ps1 | 42 +++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 azure-pipelines/Get-CodeCovTool.ps1 create mode 100644 azure-pipelines/publish-CodeCov.ps1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4db43f53..cde83cd9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,7 +109,8 @@ jobs: path: ${{ runner.temp }}/_artifacts/deployables if: always() - name: Publish code coverage results to codecov.io - run: bash <(curl -s https://codecov.io/bash) - shell: bash + run: ./azure-pipelines/publish-CodeCov.ps1 -CodeCovToken "${{ env.codecov_token }}" -PathToCodeCoverage "${{ runner.temp }}/_artifacts/coverageResults" -Name "${{ runner.os }} Coverage Results" -Flags "${{ runner.os }}Host,${{ env.BUILDCONFIGURATION }}" + shell: pwsh timeout-minutes: 3 continue-on-error: true + if: always() diff --git a/azure-pipelines/Get-CodeCovTool.ps1 b/azure-pipelines/Get-CodeCovTool.ps1 new file mode 100644 index 00000000..d6ca157b --- /dev/null +++ b/azure-pipelines/Get-CodeCovTool.ps1 @@ -0,0 +1,38 @@ +<# +.SYNOPSIS + Downloads the CodeCov.io uploader tool and returns the path to it. +#> +[CmdletBinding()] +Param( +) + +$toolsPath = & "$PSScriptRoot\Get-TempToolsPath.ps1" +$binaryToolsPath = Join-Path $toolsPath codecov +if (!(Test-Path $binaryToolsPath)) { $null = mkdir $binaryToolsPath } + +if ($IsMacOS) +{ + $codeCovPath = Join-Path $binaryToolsPath codecov + $codeCovUrl = "https://uploader.codecov.io/latest/macos/codecov" +} +elseif ($IsLinux) +{ + $codeCovPath = Join-Path $binaryToolsPath codecov + $codeCovUrl = "https://uploader.codecov.io/latest/linux/codecov" +} +else +{ + $codeCovPath = Join-Path $binaryToolsPath codecov.exe + $codeCovUrl = "https://uploader.codecov.io/latest/windows/codecov.exe" +} + +if (!(Test-Path $codeCovPath)) { + Write-Host "Downloading latest codeconv..." -ForegroundColor Yellow + (New-Object System.Net.WebClient).DownloadFile($codeCovUrl, $codeCovPath) +} + +if ($IsMacOS -or $IsLinux) { + chmod u+x $codeCovPath +} + +return (Resolve-Path $codeCovPath).Path diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index 1d7d9de7..a461c147 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -16,7 +16,10 @@ steps: displayName: ๐Ÿ“ข Publish artifacts condition: succeededOrFailed() -- bash: bash <(curl -s https://codecov.io/bash) +- powershell: | + $ArtifactStagingFolder = & "azure-pipelines/Get-ArtifactsStagingDirectory.ps1" + $CoverageResultsFolder = Join-Path $ArtifactStagingFolder "coverageResults-$(Agent.JobName)" + azure-pipelines/publish-CodeCov.ps1 -CodeCovToken "$(codecov_token)" -PathToCodeCoverage "$CoverageResultsFolder" -Name "$(Agent.JobName) Coverage Results" -Flags "$(Agent.JobName)Host,$(BuildConfiguration)" displayName: ๐Ÿ“ข Publish code coverage results to codecov.io condition: ne(variables['codecov_token'], '') timeoutInMinutes: 3 diff --git a/azure-pipelines/publish-CodeCov.ps1 b/azure-pipelines/publish-CodeCov.ps1 new file mode 100644 index 00000000..ec550e7a --- /dev/null +++ b/azure-pipelines/publish-CodeCov.ps1 @@ -0,0 +1,42 @@ +<# +.SYNOPSIS + Uploads code coverage to codecov.io +.PARAMETER CodeCovToken + Code coverage token to use +.PARAMETER PathToCodeCoverage + Path to root of code coverage files +.PARAMETER Name + Optional name to upload with codecoverge +.PARAMETER Flags + Optional flags to upload with codecoverge +#> +[CmdletBinding()] +Param ( + [string]$CodeCovToken, + [string]$PathToCodeCoverage, + [string]$Name="", + [string]$Flags="" +) + +$RepoRoot = (Resolve-Path "$PSScriptRoot/..").Path +$CodeCoveragePathWildcard = (Join-Path $PathToCodeCoverage "*.cobertura.xml") + +Write-Host "RepoRoot: $RepoRoot" -ForegroundColor Yellow +Write-Host "CodeCoveragePathWildcard: $CodeCoveragePathWildcard" -ForegroundColor Yellow + +Get-ChildItem -Recurse -Path $CodeCoveragePathWildcard | % { + + if ($IsMacOS -or $IsLinux) + { + $relativeFilePath = Resolve-Path -relative $_ + } + else + { + $relativeFilePath = Resolve-Path -relative (Get-ChildItem $_ | Select-Object -ExpandProperty Target) + } + + Write-Host "Uploading: $relativeFilePath" -ForegroundColor Yellow + Write-Host "Flags: $Flags$TargetFrameworkFlag$TestTypeFlag$OSTypeFlag" -ForegroundColor Yellow + + & (& "$PSScriptRoot/Get-CodeCovTool.ps1") -t "$CodeCovToken" -f "$relativeFilePath" -R "$RepoRoot" -F "$Flags" -n "$Name" +} From 79a441098f323457a8033a09a843300b00631de3 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 15 Jun 2022 13:08:50 -0600 Subject: [PATCH 72/99] Add completions for dotnet-test-cloud script --- azure-pipelines/dotnet-test-cloud.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/dotnet-test-cloud.ps1 b/azure-pipelines/dotnet-test-cloud.ps1 index 24bf812a..d07f1ad8 100755 --- a/azure-pipelines/dotnet-test-cloud.ps1 +++ b/azure-pipelines/dotnet-test-cloud.ps1 @@ -16,6 +16,7 @@ #> [CmdletBinding()] Param( + [ValidateSet('Debug', 'Release')] [string]$Configuration='Debug', [string]$Agent='Local', [switch]$PublishResults, From 85d970c258bafda482b003ba79f4510a491a570d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 15 Jun 2022 13:09:08 -0600 Subject: [PATCH 73/99] Move coverage merge script to a ps1 that can be locally invoked --- .gitignore | 1 + azure-pipelines/Merge-CodeCoverage.ps1 | 42 ++++++++++++++++++++++++ azure-pipelines/publish-codecoverage.yml | 14 +------- 3 files changed, 44 insertions(+), 13 deletions(-) create mode 100644 azure-pipelines/Merge-CodeCoverage.ps1 diff --git a/.gitignore b/.gitignore index 65f94776..3bb49919 100644 --- a/.gitignore +++ b/.gitignore @@ -140,6 +140,7 @@ _TeamCity* # Visual Studio code coverage results *.coverage *.coveragexml +/coveragereport/ # NCrunch _NCrunch_* diff --git a/azure-pipelines/Merge-CodeCoverage.ps1 b/azure-pipelines/Merge-CodeCoverage.ps1 new file mode 100644 index 00000000..6672b94c --- /dev/null +++ b/azure-pipelines/Merge-CodeCoverage.ps1 @@ -0,0 +1,42 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS + Merges code coverage reports. +.PARAMETER Path + The path(s) to search for Cobertura code coverage reports. +.PARAMETER Format + The format for the merged result. The default is Cobertura +.PARAMETER OutputDir + The directory the merged result will be written to. The default is `coveragereport` in the root of this repo. +#> +[CmdletBinding()] +Param( + [Parameter(Mandatory=$true)] + [string[]]$Path, + [ValidateSet('Badges', 'Clover', 'Cobertura', 'CsvSummary', 'Html', 'Html_Dark', 'Html_Light', 'HtmlChart', 'HtmlInline', 'HtmlInline_AzurePipelines', 'HtmlInline_AzurePipelines_Dark', 'HtmlInline_AzurePipelines_Light', 'HtmlSummary', 'JsonSummary', 'Latex', 'LatexSummary', 'lcov', 'MarkdownSummary', 'MHtml', 'PngChart', 'SonarQube', 'TeamCitySummary', 'TextSummary', 'Xml', 'XmlSummary')] + [string]$Format='Cobertura', + [string]$OutputDir=("$PSScriptRoot/../coveragereport") +) + +$RepoRoot = [string](Resolve-Path $PSScriptRoot/..) + +if (!(Test-Path $RepoRoot/obj/reportgenerator*)) { + dotnet tool install --tool-path $RepoRoot/obj dotnet-reportgenerator-globaltool --version 5.1.9 --configfile $PSScriptRoot/justnugetorg.nuget.config +} + +Write-Verbose "Searching $Path for *.cobertura.xml files" +$reports = Get-ChildItem -Recurse $Path -Filter *.cobertura.xml + +if ($reports) { + $reports |% { $_.FullName } |% { + # In addition to replacing {reporoot}, we also normalize on one kind of slash so that the report aggregates data for a file whether data was collected on Windows or not. + $content = Get-Content -Path $_ |% { [Regex]::Replace($_, '{reporoot}([^"]+)', { $RepoRoot + $args[0].groups[1].value.replace([IO.Path]::AltDirectorySeparatorChar, [IO.Path]::DirectorySeparatorChar) }) } + Set-Content -Path $_ -Value $content -Encoding UTF8 + } + + $Inputs = [string]::join(';', ($reports |% { Resolve-Path -relative $_.FullName })) + & "$RepoRoot/obj/reportgenerator" -reports:"$Inputs" -targetdir:$OutputDir -reporttypes:$Format +} else { + Write-Error "No reports found to merge." +} diff --git a/azure-pipelines/publish-codecoverage.yml b/azure-pipelines/publish-codecoverage.yml index 23e88a6e..6e7ef21b 100644 --- a/azure-pipelines/publish-codecoverage.yml +++ b/azure-pipelines/publish-codecoverage.yml @@ -15,19 +15,7 @@ steps: displayName: ๐Ÿ”ป Download macOS code coverage results continueOnError: true condition: ${{ parameters.includeMacOS }} -- powershell: | - dotnet tool install --tool-path obj dotnet-reportgenerator-globaltool --version 5.1.9 --configfile azure-pipelines/justnugetorg.nuget.config - Copy-Item -Recurse $(Pipeline.Workspace)/coverageResults-Windows/obj/* $(System.DefaultWorkingDirectory)/obj - Write-Host 'Substituting {reporoot} with $(System.DefaultWorkingDirectory)' - $reports = Get-ChildItem -Recurse '$(Pipeline.Workspace)/*.cobertura.xml' - $reports |% { - # In addition to replacing {reporoot}, we also normalize on one kind of slash so that the report aggregates data for a file whether data was collected on Windows or not. - $content = Get-Content -Path $_ |% { [Regex]::Replace($_, '{reporoot}([^"]+)', { '$(System.DefaultWorkingDirectory)' + $args[0].groups[1].value.replace([IO.Path]::AltDirectorySeparatorChar, [IO.Path]::DirectorySeparatorChar) }) } - Set-Content -Path $_ -Value $content -Encoding UTF8 - } - - $Inputs = [string]::join(';', ($reports |% { Resolve-Path -relative $_ })) - obj/reportgenerator -reports:"$Inputs" -targetdir:coveragereport -reporttypes:Cobertura +- powershell: azure-pipelines/Merge-CodeCoverage.ps1 -Path '$(Pipeline.Workspace)' -OutputDir coveragereport -Format Cobertura -Verbose displayName: โš™ Merge coverage - task: PublishCodeCoverageResults@1 displayName: ๐Ÿ“ข Publish code coverage results to Azure DevOps From ac247b8566c88e201f1df92013048ce7f3a30772 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 17 Jun 2022 15:44:22 -0600 Subject: [PATCH 74/99] Revert "Add completions for dotnet-test-cloud script" It was problematic since it disallowed non-default config names. This reverts commit 79a441098f323457a8033a09a843300b00631de3. --- azure-pipelines/dotnet-test-cloud.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines/dotnet-test-cloud.ps1 b/azure-pipelines/dotnet-test-cloud.ps1 index d07f1ad8..24bf812a 100755 --- a/azure-pipelines/dotnet-test-cloud.ps1 +++ b/azure-pipelines/dotnet-test-cloud.ps1 @@ -16,7 +16,6 @@ #> [CmdletBinding()] Param( - [ValidateSet('Debug', 'Release')] [string]$Configuration='Debug', [string]$Agent='Local', [switch]$PublishResults, From 6156061d6576a32b150393a22787c24629ef1545 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sat, 18 Jun 2022 12:15:07 -0600 Subject: [PATCH 75/99] Touch-up formatting and simplify/correct a bit --- azure-pipelines/Get-CodeCovTool.ps1 | 11 ++++------- azure-pipelines/publish-CodeCov.ps1 | 30 +++++++++-------------------- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/azure-pipelines/Get-CodeCovTool.ps1 b/azure-pipelines/Get-CodeCovTool.ps1 index d6ca157b..63b5d3f3 100644 --- a/azure-pipelines/Get-CodeCovTool.ps1 +++ b/azure-pipelines/Get-CodeCovTool.ps1 @@ -10,24 +10,21 @@ $toolsPath = & "$PSScriptRoot\Get-TempToolsPath.ps1" $binaryToolsPath = Join-Path $toolsPath codecov if (!(Test-Path $binaryToolsPath)) { $null = mkdir $binaryToolsPath } -if ($IsMacOS) -{ +if ($IsMacOS) { $codeCovPath = Join-Path $binaryToolsPath codecov $codeCovUrl = "https://uploader.codecov.io/latest/macos/codecov" } -elseif ($IsLinux) -{ +elseif ($IsLinux) { $codeCovPath = Join-Path $binaryToolsPath codecov $codeCovUrl = "https://uploader.codecov.io/latest/linux/codecov" } -else -{ +else { $codeCovPath = Join-Path $binaryToolsPath codecov.exe $codeCovUrl = "https://uploader.codecov.io/latest/windows/codecov.exe" } if (!(Test-Path $codeCovPath)) { - Write-Host "Downloading latest codeconv..." -ForegroundColor Yellow + Write-Host "Downloading latest codecov upload tool..." -ForegroundColor Yellow (New-Object System.Net.WebClient).DownloadFile($codeCovUrl, $codeCovPath) } diff --git a/azure-pipelines/publish-CodeCov.ps1 b/azure-pipelines/publish-CodeCov.ps1 index ec550e7a..1f4d3790 100644 --- a/azure-pipelines/publish-CodeCov.ps1 +++ b/azure-pipelines/publish-CodeCov.ps1 @@ -6,37 +6,25 @@ .PARAMETER PathToCodeCoverage Path to root of code coverage files .PARAMETER Name - Optional name to upload with codecoverge + Name to upload with codecoverge .PARAMETER Flags - Optional flags to upload with codecoverge + Flags to upload with codecoverge #> [CmdletBinding()] Param ( + [Parameter(Mandatory=$true)] [string]$CodeCovToken, + [Parameter(Mandatory=$true)] [string]$PathToCodeCoverage, - [string]$Name="", - [string]$Flags="" + [string]$Name, + [string]$Flags ) $RepoRoot = (Resolve-Path "$PSScriptRoot/..").Path -$CodeCoveragePathWildcard = (Join-Path $PathToCodeCoverage "*.cobertura.xml") -Write-Host "RepoRoot: $RepoRoot" -ForegroundColor Yellow -Write-Host "CodeCoveragePathWildcard: $CodeCoveragePathWildcard" -ForegroundColor Yellow - -Get-ChildItem -Recurse -Path $CodeCoveragePathWildcard | % { - - if ($IsMacOS -or $IsLinux) - { - $relativeFilePath = Resolve-Path -relative $_ - } - else - { - $relativeFilePath = Resolve-Path -relative (Get-ChildItem $_ | Select-Object -ExpandProperty Target) - } +Get-ChildItem -Recurse -Path $PathToCodeCoverage -Filter "*.cobertura.xml" | % { + $relativeFilePath = Resolve-Path -relative $_ Write-Host "Uploading: $relativeFilePath" -ForegroundColor Yellow - Write-Host "Flags: $Flags$TargetFrameworkFlag$TestTypeFlag$OSTypeFlag" -ForegroundColor Yellow - - & (& "$PSScriptRoot/Get-CodeCovTool.ps1") -t "$CodeCovToken" -f "$relativeFilePath" -R "$RepoRoot" -F "$Flags" -n "$Name" + & (& "$PSScriptRoot/Get-CodeCovTool.ps1") -t $CodeCovToken -f $relativeFilePath -R $RepoRoot -F $Flags -n $Name } From 50d35dfde8113caedeca40f6c14d8b93686b3055 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sat, 18 Jun 2022 15:22:38 -0600 Subject: [PATCH 76/99] fixup --- azure-pipelines/publish-CodeCov.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/publish-CodeCov.ps1 b/azure-pipelines/publish-CodeCov.ps1 index 1f4d3790..9926f018 100644 --- a/azure-pipelines/publish-CodeCov.ps1 +++ b/azure-pipelines/publish-CodeCov.ps1 @@ -23,7 +23,7 @@ Param ( $RepoRoot = (Resolve-Path "$PSScriptRoot/..").Path Get-ChildItem -Recurse -Path $PathToCodeCoverage -Filter "*.cobertura.xml" | % { - $relativeFilePath = Resolve-Path -relative $_ + $relativeFilePath = Resolve-Path -relative $_.FullName Write-Host "Uploading: $relativeFilePath" -ForegroundColor Yellow & (& "$PSScriptRoot/Get-CodeCovTool.ps1") -t $CodeCovToken -f $relativeFilePath -R $RepoRoot -F $Flags -n $Name From 8abd437547653f0c2a01eb96dcf75d9945829cf0 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sat, 18 Jun 2022 14:54:27 -0600 Subject: [PATCH 77/99] Add signature validation --- azure-pipelines/Get-CodeCovTool.ps1 | 77 ++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/azure-pipelines/Get-CodeCovTool.ps1 b/azure-pipelines/Get-CodeCovTool.ps1 index 63b5d3f3..ca580b4d 100644 --- a/azure-pipelines/Get-CodeCovTool.ps1 +++ b/azure-pipelines/Get-CodeCovTool.ps1 @@ -1,35 +1,86 @@ <# .SYNOPSIS Downloads the CodeCov.io uploader tool and returns the path to it. +.PARAMETER AllowSkipVerify + Allows skipping signature verification of the downloaded tool if gpg is not installed. #> [CmdletBinding()] Param( + [switch]$AllowSkipVerify ) -$toolsPath = & "$PSScriptRoot\Get-TempToolsPath.ps1" -$binaryToolsPath = Join-Path $toolsPath codecov -if (!(Test-Path $binaryToolsPath)) { $null = mkdir $binaryToolsPath } - if ($IsMacOS) { - $codeCovPath = Join-Path $binaryToolsPath codecov $codeCovUrl = "https://uploader.codecov.io/latest/macos/codecov" + $toolName = 'codecov' } elseif ($IsLinux) { - $codeCovPath = Join-Path $binaryToolsPath codecov $codeCovUrl = "https://uploader.codecov.io/latest/linux/codecov" + $toolName = 'codecov' } else { - $codeCovPath = Join-Path $binaryToolsPath codecov.exe $codeCovUrl = "https://uploader.codecov.io/latest/windows/codecov.exe" + $toolName = 'codecov.exe' } -if (!(Test-Path $codeCovPath)) { - Write-Host "Downloading latest codecov upload tool..." -ForegroundColor Yellow - (New-Object System.Net.WebClient).DownloadFile($codeCovUrl, $codeCovPath) +$shaSuffix = ".SHA256SUM" +$sigSuffix = $shaSuffix + ".sig" + +Function Get-FileFromWeb([Uri]$Uri, $OutDir) { + $OutFile = Join-Path $OutDir $Uri.Segments[-1] + if (!(Test-Path $OutFile)) { + Write-Verbose "Downloading $Uri..." + if (!(Test-Path $OutDir)) { New-Item -ItemType Directory -Path $OutDir | Out-Null } + try { + (New-Object System.Net.WebClient).DownloadFile($Uri, $OutFile) + } finally { + # This try/finally causes the script to abort + } + } + + $OutFile } -if ($IsMacOS -or $IsLinux) { - chmod u+x $codeCovPath +$toolsPath = & "$PSScriptRoot\Get-TempToolsPath.ps1" +$binaryToolsPath = Join-Path $toolsPath codecov +$testingPath = Join-Path $binaryToolsPath unverified +$finalToolPath = Join-Path $binaryToolsPath $toolName + +if (!(Test-Path $finalToolPath)) { + if (Test-Path $testingPath) { + Remove-Item -Recurse -Force $testingPath # ensure we download all matching files + } + $tool = Get-FileFromWeb $codeCovUrl $testingPath + $sha = Get-FileFromWeb "$codeCovUrl$shaSuffix" $testingPath + $sig = Get-FileFromWeb "$codeCovUrl$sigSuffix" $testingPath + $key = Get-FileFromWeb https://keybase.io/codecovsecurity/pgp_keys.asc $testingPath + + if ((Get-Command gpg -ErrorAction SilentlyContinue)) { + Write-Host "Importing codecov key" -ForegroundColor Yellow + gpg --import $key + Write-Host "Verifying signature on codecov hash" -ForegroundColor Yellow + gpg --verify $sig $sha + } else { + if ($AllowSkipVerify) { + Write-Warning "gpg not found. Unable to verify hash signature." + } else { + throw "gpg not found. Unable to verify hash signature. Install gpg or add -AllowSkipVerify to override." + } + } + + Write-Host "Verifying hash on downloaded tool" -ForegroundColor Yellow + $actualHash = (Get-FileHash -Path $tool -Algorithm SHA256).Hash + $expectedHash = (Get-Content $sha).Split()[0] + if ($actualHash -ne $expectedHash) { + # Validation failed. Delete the tool so we can't execute it. + #Remove-Item $codeCovPath + throw "codecov uploader tool failed signature validation." + } + + Copy-Item $tool $finalToolPath + + if ($IsMacOS -or $IsLinux) { + chmod u+x $finalToolPath + } } -return (Resolve-Path $codeCovPath).Path +return $finalToolPath From d0b0b55b461068eb2b7cc3c5e440880cf5203b75 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Sat, 18 Jun 2022 17:57:28 -0600 Subject: [PATCH 78/99] Fix perf scaling issue in merging coverage reports --- azure-pipelines/Merge-CodeCoverage.ps1 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/Merge-CodeCoverage.ps1 b/azure-pipelines/Merge-CodeCoverage.ps1 index 6672b94c..91ab67ab 100644 --- a/azure-pipelines/Merge-CodeCoverage.ps1 +++ b/azure-pipelines/Merge-CodeCoverage.ps1 @@ -31,8 +31,12 @@ $reports = Get-ChildItem -Recurse $Path -Filter *.cobertura.xml if ($reports) { $reports |% { $_.FullName } |% { # In addition to replacing {reporoot}, we also normalize on one kind of slash so that the report aggregates data for a file whether data was collected on Windows or not. - $content = Get-Content -Path $_ |% { [Regex]::Replace($_, '{reporoot}([^"]+)', { $RepoRoot + $args[0].groups[1].value.replace([IO.Path]::AltDirectorySeparatorChar, [IO.Path]::DirectorySeparatorChar) }) } - Set-Content -Path $_ -Value $content -Encoding UTF8 + $xml = [xml](Get-Content -Path $_) + $xml.coverage.packages.package.classes.class |? { $_.filename} |% { + $_.filename = $_.filename.Replace('{reporoot}', $RepoRoot).Replace([IO.Path]::AltDirectorySeparatorChar, [IO.Path]::DirectorySeparatorChar) + } + + $xml.Save($_) } $Inputs = [string]::join(';', ($reports |% { Resolve-Path -relative $_.FullName })) From fa0ac399504747ba4f8b71bfab31a974698811a6 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 20 Jun 2022 08:44:04 -0600 Subject: [PATCH 79/99] Log _pipelines scripts verbosely --- azure-pipelines/artifacts/_all.ps1 | 1 + azure-pipelines/artifacts/_pipelines.ps1 | 8 ++++++-- azure-pipelines/artifacts/_stage_all.ps1 | 10 +++++++--- azure-pipelines/artifacts/testResults.ps1 | 4 ++++ azure-pipelines/dotnet.yml | 2 +- azure-pipelines/variables/_all.ps1 | 11 +++++++++-- azure-pipelines/variables/_pipelines.ps1 | 15 +++++++++++---- 7 files changed, 39 insertions(+), 12 deletions(-) diff --git a/azure-pipelines/artifacts/_all.ps1 b/azure-pipelines/artifacts/_all.ps1 index c9182a45..033cc87c 100755 --- a/azure-pipelines/artifacts/_all.ps1 +++ b/azure-pipelines/artifacts/_all.ps1 @@ -15,6 +15,7 @@ Executes artifact scripts even if they have already been staged. #> +[CmdletBinding(SupportsShouldProcess = $true)] param ( [string]$ArtifactNameSuffix, [switch]$Force diff --git a/azure-pipelines/artifacts/_pipelines.ps1 b/azure-pipelines/artifacts/_pipelines.ps1 index a62d2675..2d3338b2 100644 --- a/azure-pipelines/artifacts/_pipelines.ps1 +++ b/azure-pipelines/artifacts/_pipelines.ps1 @@ -1,6 +1,10 @@ -# This script translates all the artifacts described by _all.ps1 -# into commands that instruct Azure Pipelines to actually collect those artifacts. +<# +.SYNOPSIS + This script translates all the artifacts described by _all.ps1 + into commands that instruct Azure Pipelines to actually collect those artifacts. +#> +[CmdletBinding()] param ( [string]$ArtifactNameSuffix, [switch]$StageOnly diff --git a/azure-pipelines/artifacts/_stage_all.ps1 b/azure-pipelines/artifacts/_stage_all.ps1 index b7166b4e..d81d16d4 100644 --- a/azure-pipelines/artifacts/_stage_all.ps1 +++ b/azure-pipelines/artifacts/_stage_all.ps1 @@ -1,6 +1,10 @@ -# This script links all the artifacts described by _all.ps1 -# into a staging directory, reading for uploading to a cloud build artifact store. -# It returns a sequence of objects with Name and Path properties. +<# +.SYNOPSIS + This script links all the artifacts described by _all.ps1 + into a staging directory, reading for uploading to a cloud build artifact store. + It returns a sequence of objects with Name and Path properties. +#> + [CmdletBinding()] param ( [string]$ArtifactNameSuffix diff --git a/azure-pipelines/artifacts/testResults.ps1 b/azure-pipelines/artifacts/testResults.ps1 index 2f894c97..4fa0fe62 100644 --- a/azure-pipelines/artifacts/testResults.ps1 +++ b/azure-pipelines/artifacts/testResults.ps1 @@ -1,3 +1,7 @@ +[CmdletBinding()] +Param( +) + $result = @{} if ($env:AGENT_TEMPDIRECTORY) { diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index a461c147..ca9fc9b8 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -11,7 +11,7 @@ steps: displayName: โš™ Update pipeline variables based on build outputs condition: succeededOrFailed() -- powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)" +- powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)" -Verbose failOnStderr: true displayName: ๐Ÿ“ข Publish artifacts condition: succeededOrFailed() diff --git a/azure-pipelines/variables/_all.ps1 b/azure-pipelines/variables/_all.ps1 index 0407d307..cc6e8810 100755 --- a/azure-pipelines/variables/_all.ps1 +++ b/azure-pipelines/variables/_all.ps1 @@ -1,7 +1,14 @@ #!/usr/bin/env pwsh -# This script returns a hashtable of build variables that should be set -# at the start of a build or release definition's execution. +<# +.SYNOPSIS + This script returns a hashtable of build variables that should be set + at the start of a build or release definition's execution. +#> + +[CmdletBinding(SupportsShouldProcess = $true)] +param ( +) $vars = @{} diff --git a/azure-pipelines/variables/_pipelines.ps1 b/azure-pipelines/variables/_pipelines.ps1 index 867b7fc8..951106d2 100644 --- a/azure-pipelines/variables/_pipelines.ps1 +++ b/azure-pipelines/variables/_pipelines.ps1 @@ -1,8 +1,15 @@ -# This script translates the variables returned by the _all.ps1 script -# into commands that instruct Azure Pipelines to actually set those variables for other pipeline tasks to consume. +<# +.SYNOPSIS + This script translates the variables returned by the _all.ps1 script + into commands that instruct Azure Pipelines to actually set those variables for other pipeline tasks to consume. -# The build or release definition may have set these variables to override -# what the build would do. So only set them if they have not already been set. + The build or release definition may have set these variables to override + what the build would do. So only set them if they have not already been set. +#> + +[CmdletBinding()] +param ( +) (& "$PSScriptRoot\_all.ps1").GetEnumerator() |% { # Always use ALL CAPS for env var names since Azure Pipelines converts variable names to all caps and on non-Windows OS, env vars are case sensitive. From cb730eacdd53e4ca06b7f9c74b06da74f44db3a9 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 20 Jun 2022 08:46:30 -0600 Subject: [PATCH 80/99] Collect test results from the right directory The dump files used to be written to an agent TEMP directory. Somewhere along the lines they moved to the TestResults directory within the repo itself. Fixes AArnott/Library.Template#167 --- azure-pipelines/artifacts/testResults.ps1 | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/azure-pipelines/artifacts/testResults.ps1 b/azure-pipelines/artifacts/testResults.ps1 index 4fa0fe62..301a4376 100644 --- a/azure-pipelines/artifacts/testResults.ps1 +++ b/azure-pipelines/artifacts/testResults.ps1 @@ -4,17 +4,8 @@ Param( $result = @{} -if ($env:AGENT_TEMPDIRECTORY) { - # The DotNetCoreCLI uses an alternate location to publish these files - $guidRegex = '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$' - $result[$env:AGENT_TEMPDIRECTORY] = (Get-ChildItem $env:AGENT_TEMPDIRECTORY -Directory |? { $_.Name -match $guidRegex } |% { - Get-ChildItem "$($_.FullName)\dotnet*.dmp","$($_.FullName)\*_crashdump.dmp","$($_.FullName)\testhost*.dmp","$($_.FullName)\Sequence_*.xml" -Recurse - }); -} -else { - $testRoot = Resolve-Path "$PSScriptRoot\..\..\test" - $result[$testRoot] = (Get-ChildItem "$testRoot\TestResults" -Recurse -Directory | Get-ChildItem -Recurse -File) -} +$testRoot = Resolve-Path "$PSScriptRoot\..\..\test" +$result[$testRoot] = (Get-ChildItem "$testRoot\TestResults" -Recurse -Directory | Get-ChildItem -Recurse -File) $testlogsPath = "$env:BUILD_ARTIFACTSTAGINGDIRECTORY\test_logs" if (Test-Path $testlogsPath) { From 540ea6a9b78c8c2a8e99ffb553ba523defaa7e81 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 20 Jun 2022 16:25:12 -0600 Subject: [PATCH 81/99] Prepare release notes with breaking changes --- azure-pipelines/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index 2679b24e..4c9db860 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -51,6 +51,7 @@ jobs: changeLogType: issueBased changeLogLabels: | [ + { "label" : "breaking change", "displayName" : "Breaking changes", "state" : "closed" }, { "label" : "bug", "displayName" : "Fixes", "state" : "closed" }, { "label" : "enhancement", "displayName": "Enhancements", "state" : "closed" } ] From 2c7486f63461a1064a4b8454dc52a289377a73d9 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 22 Jun 2022 19:34:34 -0600 Subject: [PATCH 82/99] Handle 4-integer package versions in InsertConfigValues.ps1 --- azure-pipelines/variables/InsertConfigValues.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/variables/InsertConfigValues.ps1 b/azure-pipelines/variables/InsertConfigValues.ps1 index efed018c..1ab53f34 100644 --- a/azure-pipelines/variables/InsertConfigValues.ps1 +++ b/azure-pipelines/variables/InsertConfigValues.ps1 @@ -3,7 +3,7 @@ $InsertedPkgs = (& "$PSScriptRoot\..\artifacts\VSInsertion.ps1" -SbomNotRequired $icv=@() foreach ($kvp in $InsertedPkgs.GetEnumerator()) { $kvp.Value |% { - if ($_.Name -match "^(.*)\.(\d+\.\d+\.\d+(?:-.*?)?)(?:\.symbols)?\.nupkg$") { + if ($_.Name -match "^(.*?)\.(\d+\.\d+\.\d+(?:\.\d+)?(?:-.*?)?)(?:\.symbols)?\.nupkg$") { $id = $Matches[1] $version = $Matches[2] $icv += "$id=$version" From c5683bea1c776521445818e010a656a96d58b873 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 21 Jul 2022 21:14:22 -0600 Subject: [PATCH 83/99] Fix symbol file collection when the dll or exe is missing --- azure-pipelines/Get-SymbolFiles.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/Get-SymbolFiles.ps1 b/azure-pipelines/Get-SymbolFiles.ps1 index fccb1bb1..9183c340 100644 --- a/azure-pipelines/Get-SymbolFiles.ps1 +++ b/azure-pipelines/Get-SymbolFiles.ps1 @@ -49,8 +49,13 @@ $PDBs |% { $BinaryImagePath = $dllPath } elseif (Test-Path $exePath) { $BinaryImagePath = $exePath + } else { + Write-Warning "`"$_`" found with no matching binary file." + $BinaryImagePath = $null } - Write-Output $BinaryImagePath - Write-Output $_.FullName + if ($BinaryImagePath) { + Write-Output $BinaryImagePath + Write-Output $_.FullName + } } From 540833819f90b3dacf9753c7c2f3baee19e5094c Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 21 Jul 2022 18:18:26 -0600 Subject: [PATCH 84/99] Fix publishing of legacy symbols where on binary image can be found --- azure-pipelines/Publish-Legacy-Symbols.ps1 | 25 +++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/azure-pipelines/Publish-Legacy-Symbols.ps1 b/azure-pipelines/Publish-Legacy-Symbols.ps1 index 6ca7ddb6..5c4035a1 100644 --- a/azure-pipelines/Publish-Legacy-Symbols.ps1 +++ b/azure-pipelines/Publish-Legacy-Symbols.ps1 @@ -14,19 +14,24 @@ Get-ChildItem "$ArtifactStagingFolder\*.pdb" -Recurse |% { $BinaryImagePath = $dllPath } elseif (Test-Path $exePath) { $BinaryImagePath = $exePath + } else { + Write-Warning "`"$_`" found with no matching binary file." + $BinaryImagePath = $null } - # Convert the PDB to legacy Windows PDBs - Write-Host "Converting PDB for $_" -ForegroundColor DarkGray - $WindowsPdbDir = "$($_.Directory.FullName)\$WindowsPdbSubDirName" - if (!(Test-Path $WindowsPdbDir)) { mkdir $WindowsPdbDir | Out-Null } - $legacyPdbPath = "$WindowsPdbDir\$($_.BaseName).pdb" - & "$PSScriptRoot\Convert-PDB.ps1" -DllPath $BinaryImagePath -PdbPath $_ -OutputPath $legacyPdbPath - if ($LASTEXITCODE -ne 0) { - Write-Warning "PDB conversion of `"$_`" failed." - } + if ($BinaryImagePath) { + # Convert the PDB to legacy Windows PDBs + Write-Host "Converting PDB for $_" -ForegroundColor DarkGray + $WindowsPdbDir = "$($_.Directory.FullName)\$WindowsPdbSubDirName" + if (!(Test-Path $WindowsPdbDir)) { mkdir $WindowsPdbDir | Out-Null } + $legacyPdbPath = "$WindowsPdbDir\$($_.BaseName).pdb" + & "$PSScriptRoot\Convert-PDB.ps1" -DllPath $BinaryImagePath -PdbPath $_ -OutputPath $legacyPdbPath + if ($LASTEXITCODE -ne 0) { + Write-Warning "PDB conversion of `"$_`" failed." + } - Move-Item $legacyPdbPath $_ -Force + Move-Item $legacyPdbPath $_ -Force + } } Write-Host "##vso[artifact.upload containerfolder=symbols-legacy;artifactname=symbols-legacy;]$ArtifactStagingFolder" From b5f7042d435965c623a41856c911675c7ffb5ad1 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 21 Jul 2022 18:23:12 -0600 Subject: [PATCH 85/99] Update SBOM support to use the new SBOM plugin Remove deprecated SBOM handling SBOM is now generated as part of the build (when it applies). --- azure-pipelines/Install-NuGetPackage.ps1 | 2 ++ azure-pipelines/artifacts/VSInsertion.ps1 | 11 ----------- azure-pipelines/build.yml | 5 ----- azure-pipelines/microbuild.after.yml | 9 --------- azure-pipelines/microbuild.before.yml | 3 +++ azure-pipelines/variables/InsertConfigValues.ps1 | 2 +- init.ps1 | 12 ++++++++++++ 7 files changed, 18 insertions(+), 26 deletions(-) diff --git a/azure-pipelines/Install-NuGetPackage.ps1 b/azure-pipelines/Install-NuGetPackage.ps1 index 0bf05710..13967233 100644 --- a/azure-pipelines/Install-NuGetPackage.ps1 +++ b/azure-pipelines/Install-NuGetPackage.ps1 @@ -45,6 +45,8 @@ try { $p = Start-Process $nugetPath $nugetArgs -NoNewWindow -Wait -PassThru if ($p.ExitCode -ne 0) { throw } } + + Write-Output (Get-ChildItem "$PackagesDir\$PackageId.*")[0].FullName } finally { Pop-Location } diff --git a/azure-pipelines/artifacts/VSInsertion.ps1 b/azure-pipelines/artifacts/VSInsertion.ps1 index 0d1cb373..ba6af320 100644 --- a/azure-pipelines/artifacts/VSInsertion.ps1 +++ b/azure-pipelines/artifacts/VSInsertion.ps1 @@ -1,12 +1,7 @@ # This artifact captures everything needed to insert into VS (NuGet packages, insertion metadata, etc.) -<# -.PARAMETER SbomNotRequired - Indicates that returning the artifacts available is preferable to nothing at all when the SBOM has not yet been generated. -#> [CmdletBinding()] Param ( - [switch]$SbomNotRequired ) if ($IsMacOS -or $IsLinux) { @@ -23,12 +18,6 @@ if (!$BuildConfiguration) { $PackagesRoot = "$RepoRoot/bin/Packages/$BuildConfiguration/NuGet" -# This artifact is not ready if we're running on the devdiv AzDO account and we don't have an SBOM yet. -if ($env:SYSTEM_COLLECTIONID -eq '011b8bdf-6d56-4f87-be0d-0092136884d9' -and -not (Test-Path $PackagesRoot/_manifest) -and -not $SbomNotRequired) { - Write-Host "Skipping because SBOM isn't generated yet." - return @{} -} - if (!(Test-Path $PackagesRoot)) { Write-Warning "Skipping because packages haven't been built yet." return @{} diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index a8c33859..f0e6c7e1 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -36,11 +36,6 @@ jobs: - template: microbuild.after.yml parameters: EnableAPIScan: ${{ parameters.EnableAPIScan }} - # Repeat this step to scoop up any artifacts that would only be collected after running microbuild.after.yml - - powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)" - failOnStderr: true - displayName: Publish artifacts - condition: succeededOrFailed() - template: expand-template.yml diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 87634808..6f107833 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -13,15 +13,6 @@ steps: condition: succeededOrFailed() displayName: MicroBuild Cleanup -- task: ManifestGeneratorTask@0 - displayName: Software Bill of Materials generation - inputs: - BuildDropPath: $(System.DefaultWorkingDirectory)/bin/Library/$(BuildConfiguration) - BuildComponentPath: $(System.DefaultWorkingDirectory)/obj/src/Library - -- powershell: Copy-Item -Recurse -Verbose "$(System.DefaultWorkingDirectory)/bin/Library/$(BuildConfiguration)/_manifest" "$(System.DefaultWorkingDirectory)/bin/Packages/$(BuildConfiguration)/NuGet" - displayName: Publish Software Bill of Materials - - task: Ref12Analyze@0 displayName: Ref12 (Codex) Analyze inputs: diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index ab930694..05efd4a9 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -12,3 +12,6 @@ steps: inputs: signType: $(SignType) displayName: Install MicroBuild Signing Plugin + +- task: MicroBuildSbomPlugin@1 + displayName: Install MicroBuild Sbom Plugin diff --git a/azure-pipelines/variables/InsertConfigValues.ps1 b/azure-pipelines/variables/InsertConfigValues.ps1 index 1ab53f34..3ae11de9 100644 --- a/azure-pipelines/variables/InsertConfigValues.ps1 +++ b/azure-pipelines/variables/InsertConfigValues.ps1 @@ -1,4 +1,4 @@ -$InsertedPkgs = (& "$PSScriptRoot\..\artifacts\VSInsertion.ps1" -SbomNotRequired) +$InsertedPkgs = (& "$PSScriptRoot\..\artifacts\VSInsertion.ps1") $icv=@() foreach ($kvp in $InsertedPkgs.GetEnumerator()) { diff --git a/init.ps1 b/init.ps1 index 0a7e1cbd..4a576ac2 100755 --- a/init.ps1 +++ b/init.ps1 @@ -39,6 +39,8 @@ Install the MicroBuild setup plugin for building VSIXv3 packages. .PARAMETER OptProf Install the MicroBuild OptProf plugin for building optimized assemblies on desktop machines. +.PARAMETER Sbom + Install the MicroBuild SBOM plugin. .PARAMETER AccessToken An optional access token for authenticating to Azure Artifacts authenticated feeds. #> @@ -63,6 +65,8 @@ Param ( [Parameter()] [switch]$OptProf, [Parameter()] + [switch]$SBOM, + [Parameter()] [string]$AccessToken ) @@ -130,6 +134,14 @@ try { $EnvVars['LocLanguages'] = "JPN" } + if ($SBOM) { + Write-Host "Installing MicroBuild SBOM plugin" -ForegroundColor $HeaderColor + & $InstallNuGetPkgScriptPath MicroBuild.Plugins.Sbom -source $MicroBuildPackageSource -Verbosity $nugetVerbosity + $PkgMicrosoft_ManifestTool_CrossPlatform = & $InstallNuGetPkgScriptPath Microsoft.ManifestTool.CrossPlatform -source 'https://1essharedassets.pkgs.visualstudio.com/1esPkgs/_packaging/SBOMTool/nuget/v3/index.json' -Verbosity $nugetVerbosity + $EnvVars['GenerateSBOM'] = "true" + $EnvVars['PkgMicrosoft_ManifestTool_CrossPlatform'] = $PkgMicrosoft_ManifestTool_CrossPlatform + } + & "$PSScriptRoot/tools/Set-EnvVars.ps1" -Variables $EnvVars -PrependPath $PrependPath | Out-Null } catch { From 7cc5a15009551f5951cb64916b29db85d438433b Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 22 Jul 2022 16:57:52 -0600 Subject: [PATCH 86/99] Skip zipping sources when real signing --- azure-pipelines/microbuild.before.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index 05efd4a9..72fd72bb 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -11,6 +11,7 @@ steps: - task: MicroBuildSigningPlugin@3 inputs: signType: $(SignType) + zipSources: false displayName: Install MicroBuild Signing Plugin - task: MicroBuildSbomPlugin@1 From b7a52ae8ddfa47f86d10943528dfaafec91989b2 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 25 Jul 2022 14:51:05 -0600 Subject: [PATCH 87/99] Update MicroBuild package version --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 8886338e..c55c85c8 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -31,7 +31,7 @@ true snupkg - 2.0.68 + 2.0.72 From 7d69d2a446ba4e3cec42c9787b3c18bf96d30a26 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 26 Jul 2022 17:44:03 -0600 Subject: [PATCH 88/99] Update MicroBuild version --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index c55c85c8..bc6ba5cd 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -31,7 +31,7 @@ true snupkg - 2.0.72 + 2.0.73 From 0bb373adc92cdaca099a82d4dcdb1d59c8f73ffe Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 27 Jul 2022 13:58:05 -0600 Subject: [PATCH 89/99] Fix WPF workaround This presumably broke when the intermediate directory was changed to include the relative path to the project directory. --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 22c1b94d..dcf31a29 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -75,7 +75,7 @@ It's also not necessary to generate these assets. --> - <_WpfTempProjectNuGetFilePathNoExt>$(RepoRootPath)obj\$(_TargetAssemblyProjectName)\$(_TargetAssemblyProjectName)$(MSBuildProjectExtension).nuget.g + <_WpfTempProjectNuGetFilePathNoExt>$(BaseIntermediateOutputPath)..\$(_TargetAssemblyProjectName)\$(_TargetAssemblyProjectName)$(MSBuildProjectExtension).nuget.g false false From 3ec6f43d2500784af5585714d411cbf33d9bd55c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 12:25:45 -0600 Subject: [PATCH 90/99] Bump Nullable from 1.3.0 to 1.3.1 (#172) Bumps [Nullable](https://github.com/manuelroemer/Nullable) from 1.3.0 to 1.3.1. - [Release notes](https://github.com/manuelroemer/Nullable/releases) - [Changelog](https://github.com/manuelroemer/Nullable/blob/master/CHANGELOG.md) - [Commits](https://github.com/manuelroemer/Nullable/commits) --- updated-dependencies: - dependency-name: Nullable dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index dcf31a29..c2b4b004 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -38,7 +38,7 @@ - + From 45948958aece65a9048c74443139f5ce5fed952d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 18:33:20 +0000 Subject: [PATCH 91/99] Bump Nerdbank.GitVersioning from 3.5.107 to 3.5.108 (#171) --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index c2b4b004..d13dbc48 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -37,7 +37,7 @@ - + From 396af02e150f71f9e1fdd942807a7332c698eca5 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 2 Aug 2022 12:19:04 -0600 Subject: [PATCH 92/99] Give plenty of time for pipelines for real-signing --- azure-pipelines/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index f0e6c7e1..7ce7738a 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -11,6 +11,7 @@ parameters: jobs: - job: Windows pool: ${{ parameters.windowsPool }} + timeoutInMinutes: 180 # Give plenty of time due to real signing variables: - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: # https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/25351/APIScan-step-by-step-guide-to-setting-up-a-Pipeline From 7fd07c23d49f61f69a0e50a50ef12dc9c9bedc5b Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 2 Aug 2022 12:19:16 -0600 Subject: [PATCH 93/99] Skip APIScan on arm64 binaries --- azure-pipelines/secure-development-tools.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index c5ebf04e..3e1badc4 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -29,6 +29,7 @@ steps: # Exclude any patterns from the Contents (e.g. `!**/git2*`) that we have symbols for but do not need to run APIScan on. Contents: | ** + !**/arm64/** TargetFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs condition: and(succeeded(), ${{ parameters.EnableAPIScan }}, ne(variables.ApiScanClientId, '')) From b39494e5d1115a599f4ed01565bc5419a8dace36 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 4 Aug 2022 09:16:02 -0600 Subject: [PATCH 94/99] Make running tests in a pipeline optional --- azure-pipelines.yml | 5 +++++ azure-pipelines/build.yml | 16 +++++++++++++--- azure-pipelines/dotnet.yml | 4 ++++ azure-pipelines/publish-codecoverage.yml | 2 +- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d98ed107..2cac17bd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,6 +17,10 @@ parameters: displayName: Build on macOS type: boolean default: false # macOS is often bogged down in Azure Pipelines +- name: RunTests + displayName: Run tests + type: boolean + default: true variables: TreatWarningsAsErrors: true @@ -30,3 +34,4 @@ jobs: - template: azure-pipelines/build.yml parameters: includeMacOS: ${{ parameters.includeMacOS }} + RunTests: ${{ parameters.RunTests }} diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 5bfd95f5..cb489949 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -4,6 +4,9 @@ parameters: default: vmImage: windows-2022 - name: includeMacOS +- name: RunTests + type: boolean + default: true jobs: - job: Windows @@ -17,6 +20,8 @@ jobs: displayName: โš™ Set build number - template: dotnet.yml + parameters: + RunTests: ${{ parameters.RunTests }} - template: expand-template.yml - job: Linux @@ -27,6 +32,8 @@ jobs: clean: true - template: install-dependencies.yml - template: dotnet.yml + parameters: + RunTests: ${{ parameters.RunTests }} - template: expand-template.yml - job: macOS @@ -38,6 +45,8 @@ jobs: clean: true - template: install-dependencies.yml - template: dotnet.yml + parameters: + RunTests: ${{ parameters.RunTests }} - template: expand-template.yml - job: WrapUp @@ -56,7 +65,8 @@ jobs: - template: publish-symbols.yml parameters: includeMacOS: ${{ parameters.includeMacOS }} - - template: publish-codecoverage.yml - parameters: - includeMacOS: ${{ parameters.includeMacOS }} + - ${{ if parameters.RunTests }}: + - template: publish-codecoverage.yml + parameters: + includeMacOS: ${{ parameters.includeMacOS }} - template: publish-deployables.yml diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index ca9fc9b8..055d44e9 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -1,3 +1,6 @@ +parameters: + RunTests: + steps: - script: dotnet build -t:build,pack --no-restore -c $(BuildConfiguration) /bl:"$(Build.ArtifactStagingDirectory)/build_logs/build.binlog" @@ -5,6 +8,7 @@ steps: - powershell: azure-pipelines/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults displayName: ๐Ÿงช dotnet test + condition: and(succeeded(), ${{ parameters.RunTests }}) - powershell: azure-pipelines/variables/_pipelines.ps1 failOnStderr: true diff --git a/azure-pipelines/publish-codecoverage.yml b/azure-pipelines/publish-codecoverage.yml index 6e7ef21b..11de8ffa 100644 --- a/azure-pipelines/publish-codecoverage.yml +++ b/azure-pipelines/publish-codecoverage.yml @@ -14,7 +14,7 @@ steps: artifact: coverageResults-macOS displayName: ๐Ÿ”ป Download macOS code coverage results continueOnError: true - condition: ${{ parameters.includeMacOS }} + condition: and(succeeded(), ${{ parameters.includeMacOS }}) - powershell: azure-pipelines/Merge-CodeCoverage.ps1 -Path '$(Pipeline.Workspace)' -OutputDir coveragereport -Format Cobertura -Verbose displayName: โš™ Merge coverage - task: PublishCodeCoverageResults@1 From f68beb63687c66a3301f16bed97a791088281c22 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 4 Aug 2022 15:09:03 -0600 Subject: [PATCH 95/99] More displayName icons --- azure-pipelines/microbuild.after.yml | 6 +++--- azure-pipelines/microbuild.before.yml | 8 ++++---- azure-pipelines/secure-development-tools.yml | 18 +++++++++--------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 6f107833..d6378b9f 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -4,17 +4,17 @@ parameters: steps: - task: MicroBuildCodesignVerify@3 - displayName: Verify Signed Files + displayName: ๐Ÿ” Verify Signed Files inputs: TargetFolders: | $(Build.SourcesDirectory)/bin/Packages/$(BuildConfiguration)/NuGet - task: MicroBuildCleanup@1 condition: succeededOrFailed() - displayName: MicroBuild Cleanup + displayName: โš™๏ธ MicroBuild Cleanup - task: Ref12Analyze@0 - displayName: Ref12 (Codex) Analyze + displayName: ๐Ÿ“‘ Ref12 (Codex) Analyze inputs: codexoutputroot: $(Build.ArtifactStagingDirectory)\Codex workflowArguments: | diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index 72fd72bb..a27cf36b 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -1,9 +1,9 @@ steps: - task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection + displayName: ๐Ÿ” Component Detection - task: notice@0 - displayName: Generate NOTICE file + displayName: ๐Ÿ› ๏ธ Generate NOTICE file inputs: outputfile: $(System.DefaultWorkingDirectory)/obj/NOTICE outputformat: text @@ -12,7 +12,7 @@ steps: inputs: signType: $(SignType) zipSources: false - displayName: Install MicroBuild Signing Plugin + displayName: ๐Ÿ”ง Install MicroBuild Signing Plugin - task: MicroBuildSbomPlugin@1 - displayName: Install MicroBuild Sbom Plugin + displayName: ๐Ÿ”ง Install MicroBuild Sbom Plugin diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index 3e1badc4..324bfe86 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -5,17 +5,17 @@ parameters: steps: - task: CredScan@3 - displayName: Run CredScan + displayName: ๐Ÿ” Run CredScan - task: PoliCheck@2 - displayName: Run PoliCheck + displayName: ๐Ÿ” Run PoliCheck inputs: targetType: F targetArgument: $(System.DefaultWorkingDirectory) optionsUEPATH: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml - task: BinSkim@4 - displayName: Run BinSkim + displayName: ๐Ÿ” Run BinSkim inputs: InputType: Basic Function: analyze @@ -23,7 +23,7 @@ steps: AnalyzeTargetGlob: $(BinSkimTargets) - task: CopyFiles@2 - displayName: Collect APIScan inputs + displayName: ๐Ÿ” Collect APIScan inputs inputs: SourceFolder: $(Build.ArtifactStagingDirectory)/Symbols-$(Agent.JobName) # Exclude any patterns from the Contents (e.g. `!**/git2*`) that we have symbols for but do not need to run APIScan on. @@ -34,7 +34,7 @@ steps: condition: and(succeeded(), ${{ parameters.EnableAPIScan }}, ne(variables.ApiScanClientId, '')) - task: APIScan@2 - displayName: Run APIScan + displayName: ๐Ÿ” Run APIScan inputs: softwareFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs softwareName: $(SymbolsFeatureName) @@ -46,12 +46,12 @@ steps: AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) - task: SdtReport@2 - displayName: Create Security Analysis Report + displayName: ๐Ÿ› ๏ธ Create Security Analysis Report inputs: GdnExportAllTools: true - task: PublishSecurityAnalysisLogs@3 - displayName: Publish Code Analysis Logs + displayName: ๐Ÿ“ข Publish Code Analysis Logs inputs: ArtifactName: CodeAnalysisLogs ArtifactType: Container @@ -60,7 +60,7 @@ steps: ToolLogsNotFoundAction: Standard - task: PostAnalysis@2 - displayName: Break on compliance issues + displayName: ๐Ÿ‹๏ธโ€โ™€๏ธ Break on compliance issues inputs: GdnBreakAllTools: true GdnBreakGdnToolBinSkimSeverity: Warning @@ -72,5 +72,5 @@ steps: # This is useful when false positives appear so we can copy some of the output into the suppressions file. - publish: $(Build.ArtifactStagingDirectory)/guardian_failures_as_suppressions artifact: guardian_failures_as_suppressions - displayName: Publish Guardian failures + displayName: ๐Ÿ” Publish Guardian failures condition: failed() From 5ff626cb52ca4f05217a7392f317928c5b4deba6 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 4 Aug 2022 15:14:15 -0600 Subject: [PATCH 96/99] Add a queue-time parameter for skipping compliance tasks --- azure-pipelines/build.yml | 6 +++++- azure-pipelines/microbuild.after.yml | 11 ++++++----- azure-pipelines/official.yml | 7 ++++++- azure-pipelines/secure-development-tools.yml | 6 ++++-- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 271a3664..2face913 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -7,9 +7,12 @@ parameters: - name: RunTests type: boolean default: true +- name: EnableCompliance + type: boolean + default: true - name: EnableAPIScan type: boolean - default: false + default: true jobs: - job: Windows @@ -41,6 +44,7 @@ jobs: - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - template: microbuild.after.yml parameters: + EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} - template: expand-template.yml diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index d6378b9f..4c3554ab 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -1,6 +1,6 @@ parameters: -- name: EnableAPIScan - type: boolean + EnableCompliance: + EnableAPIScan: steps: - task: MicroBuildCodesignVerify@3 @@ -26,6 +26,7 @@ steps: condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'), ne(variables['Build.Reason'], 'PullRequest')) continueOnError: true -- template: secure-development-tools.yml - parameters: - EnableAPIScan: ${{ parameters.EnableAPIScan }} +- ${{ if eq(parameters.EnableCompliance, 'true') }}: + - template: secure-development-tools.yml + parameters: + EnableAPIScan: ${{ parameters.EnableAPIScan }} diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 1cc141f5..743fecb2 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -33,8 +33,12 @@ parameters: displayName: Run tests type: boolean default: true +- name: EnableCompliance + displayName: Run Compliance Tools + type: boolean + default: true - name: EnableAPIScan - displayName: Run APIScan + displayName: Include APIScan with Compliance tools type: boolean default: true @@ -55,6 +59,7 @@ stages: jobs: - template: build.yml parameters: + EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} windowsPool: VSEngSS-MicroBuild2022-1ES includeMacOS: ${{ parameters.includeMacOS }} diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index 324bfe86..cf5e2635 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -1,9 +1,11 @@ parameters: -- name: EnableAPIScan - type: boolean + EnableAPIScan: steps: +- powershell: echo "##vso[build.addbuildtag]compliance" + displayName: ๐Ÿท๏ธ Tag run with 'compliance' + - task: CredScan@3 displayName: ๐Ÿ” Run CredScan From 4c03a7464f2d8512b75862e3a05e26db6f24f809 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 4 Aug 2022 15:47:23 -0600 Subject: [PATCH 97/99] Fix Expand-Template --- Expand-Template.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/Expand-Template.ps1 b/Expand-Template.ps1 index 5705e722..8d4c2f4b 100755 --- a/Expand-Template.ps1 +++ b/Expand-Template.ps1 @@ -155,9 +155,6 @@ try { $YmlReplacements['(codecov_token: ).*(#.*)'] = "#`$1`$2" } Replace-Placeholders -Path "azure-pipelines.yml" -Replacements $YmlReplacements - Replace-Placeholders -Path "azure-pipelines/microbuild.after.yml" -Replacements @{ - 'Library' = $LibraryName; - } Replace-Placeholders -Path "azure-pipelines/variables/InsertVersionsValues.ps1" -Replacements @{ 'LibraryName' = $LibraryName; From 44062e8b68603f85ee53326bcd8eadbacfe2bd1d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 8 Aug 2022 16:52:57 -0600 Subject: [PATCH 98/99] Improve icon for task display name --- azure-pipelines/secure-development-tools.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index cf5e2635..ac540aea 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -48,7 +48,7 @@ steps: AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) - task: SdtReport@2 - displayName: ๐Ÿ› ๏ธ Create Security Analysis Report + displayName: ๐Ÿ“ƒ Create Security Analysis Report inputs: GdnExportAllTools: true From 94caea80e4c5f08fc6a144673ae2f4363ed6d28b Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 1 Sep 2022 17:52:50 -0600 Subject: [PATCH 99/99] Update package versions Fix build warnings from dependency updates --- Directory.Build.props | 2 +- .../GenerationSandbox.Tests.csproj | 4 ++-- .../GeneratorTests.cs | 14 +++++++------- .../Microsoft.Windows.CsWin32.Tests.csproj | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 8523d008..29ab3472 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -43,7 +43,7 @@ - + diff --git a/test/GenerationSandbox.Tests/GenerationSandbox.Tests.csproj b/test/GenerationSandbox.Tests/GenerationSandbox.Tests.csproj index e1b6a7ee..f0766fc2 100644 --- a/test/GenerationSandbox.Tests/GenerationSandbox.Tests.csproj +++ b/test/GenerationSandbox.Tests/GenerationSandbox.Tests.csproj @@ -30,9 +30,9 @@ - + - + diff --git a/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs b/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs index 6ee820cc..507e86ea 100644 --- a/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs +++ b/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs @@ -206,20 +206,20 @@ public void TemplateProvidedMembersMatchVisibilityWithContainingType_OtherMember this.CollectGeneratedCode(this.generator); this.AssertNoDiagnostics(); - StructDeclarationSyntax? pcstrType = (StructDeclarationSyntax?)this.FindGeneratedType("PCSTR").Single(); + StructDeclarationSyntax pcstrType = (StructDeclarationSyntax)this.FindGeneratedType("PCSTR").Single(); SyntaxKind expectedVisibility = generatePublic ? SyntaxKind.PublicKeyword : SyntaxKind.InternalKeyword; // Assert fields - Assert.Contains(pcstrType?.Members.OfType(), f => f.Declaration.Variables.Any(v => v.Identifier.ValueText == "Value") && f.Modifiers.Any(expectedVisibility)); + Assert.Contains(pcstrType.Members.OfType(), f => f.Declaration.Variables.Any(v => v.Identifier.ValueText == "Value") && f.Modifiers.Any(expectedVisibility)); // Assert properties - Assert.Contains(pcstrType?.Members.OfType(), p => p.Identifier.ValueText == "Length" && p.Modifiers.Any(expectedVisibility)); + Assert.Contains(pcstrType.Members.OfType(), p => p.Identifier.ValueText == "Length" && p.Modifiers.Any(expectedVisibility)); // Assert constructors - Assert.All(pcstrType?.Members.OfType(), c => c.Modifiers.Any(expectedVisibility)); + Assert.All(pcstrType.Members.OfType(), c => c.Modifiers.Any(expectedVisibility)); // Assert that private members remain private. - Assert.Contains(pcstrType?.Members.OfType(), p => p.Identifier.ValueText == "DebuggerDisplay" && p.Modifiers.Any(SyntaxKind.PrivateKeyword)); + Assert.Contains(pcstrType.Members.OfType(), p => p.Identifier.ValueText == "DebuggerDisplay" && p.Modifiers.Any(SyntaxKind.PrivateKeyword)); } [Fact] @@ -523,8 +523,8 @@ public void WinRTInterfaceWithWinRTOutObjectUsesMarshaler() AttributeSyntax marshalAsAttr = Assert.Single(FindAttribute(lastParam.AttributeLists, "MarshalAs")); Assert.True(marshalAsAttr.ArgumentList?.Arguments[0].ToString() == "UnmanagedType.CustomMarshaler"); - Assert.Single(marshalAsAttr.ArgumentList?.Arguments.Where(arg => arg.ToString() == $"MarshalCookie = \"{WinRTClassName}\"")); - Assert.Single(marshalAsAttr.ArgumentList?.Arguments.Where(arg => arg.ToString() == $"MarshalType = \"{WinRTCustomMarshalerFullName}\"")); + Assert.Single(marshalAsAttr.ArgumentList.Arguments.Where(arg => arg.ToString() == $"MarshalCookie = \"{WinRTClassName}\"")); + Assert.Single(marshalAsAttr.ArgumentList.Arguments.Where(arg => arg.ToString() == $"MarshalType = \"{WinRTCustomMarshalerFullName}\"")); // Make sure the WinRT marshaler was brought in Assert.Single(this.FindGeneratedType(WinRTCustomMarshalerClass)); diff --git a/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj b/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj index 9cdfee39..6a0bc18b 100644 --- a/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj +++ b/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj @@ -42,11 +42,11 @@ - + - +