Merge pull request #1242 from microsoft/signing2

Switch official pipeline over to devdiv and microbuild
microsoft · Jul 20, 2024 · d6bf225 · d6bf225
2 parents 03119df + 43ca372
commit d6bf225
Show file tree

Hide file tree

Showing 38 changed files with 689 additions and 228 deletions.
diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json
@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "powershell": {
-      "version": "7.4.2",
+      "version": "7.4.3",
       "commands": [
         "pwsh"
       ]
@@ -21,4 +21,4 @@
       ]
     }
   }
-}
+}
diff --git a/.gitignore b/.gitignore
@@ -352,3 +352,6 @@ MigrationBackup/
 
 # mac-created file to track user view preferences for a directory
 .DS_Store
+
+# Analysis results
+*.sarif
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -5,15 +5,14 @@
     <RepoRootPath>$(MSBuildThisFileDirectory)</RepoRootPath>
     <BaseIntermediateOutputPath>$(RepoRootPath)obj\$([MSBuild]::MakeRelative($(RepoRootPath), $(MSBuildProjectDirectory)))\</BaseIntermediateOutputPath>
     <BaseOutputPath Condition=" '$(BaseOutputPath)' == '' ">$(RepoRootPath)bin\$(MSBuildProjectName)\</BaseOutputPath>
-    <PackageOutputPath>$(RepoRootPath)bin\Packages\$(Configuration)\</PackageOutputPath>
-    <LangVersion>12</LangVersion>
+    <PackageOutputPath>$(RepoRootPath)bin\Packages\$(Configuration)\NuGet\</PackageOutputPath>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <AnalysisLevel>latest</AnalysisLevel>
     <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
-    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
-    <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
+    <ProduceReferenceAssembly>true</ProduceReferenceAssembly>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <EnableWindowsTargeting>true</EnableWindowsTargeting>
 
     <!-- https://github.com/dotnet/msbuild/blob/main/documentation/ProjectReference-Protocol.md#setplatform-negotiation -->
@@ -29,15 +28,11 @@
     <!-- Local builds should embed PDBs so we never lose them when a subsequent build occurs. -->
     <DebugType Condition=" '$(CI)' != 'true' and '$(TF_BUILD)' != 'true' ">embedded</DebugType>
 
-    <SignAssembly>true</SignAssembly>
-    <AssemblyOriginatorKeyFile>$(MSBuildThisFileDirectory)strongname.snk</AssemblyOriginatorKeyFile>
-
     <PackageProjectUrl>https://github.com/microsoft/CsWin32</PackageProjectUrl>
     <Company>Microsoft</Company>
     <Authors>Microsoft</Authors>
     <Copyright>© Microsoft Corporation. All rights reserved.</Copyright>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
-    <PackageProjectUrl>https://github.com/Microsoft/CsWin32</PackageProjectUrl>
     <PublishRepositoryUrl>true</PublishRepositoryUrl>
     <EmbedUntrackedSources>true</EmbedUntrackedSources>
     <IncludeSymbols Condition=" '$(DebugType)' != 'embedded' ">true</IncludeSymbols>

diff --git a/Directory.Build.targets b/Directory.Build.targets
@@ -1,5 +1,9 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project>
+  <PropertyGroup>
+    <LangVersion Condition="'$(Language)'=='C#'">12</LangVersion>
+    <LangVersion Condition="'$(Language)'=='VB'">16.9</LangVersion>
+  </PropertyGroup>
   <ItemGroup>
     <!-- Avoid compile error about missing namespace when combining ImplicitUsings with .NET Framework target frameworks. -->
     <Using Remove="System.Net.Http" Condition="'$(TargetFrameworkIdentifier)'=='.NETFramework'" />

diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -4,6 +4,8 @@
   <PropertyGroup>
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
     <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
+
+    <MicroBuildVersion>2.0.162</MicroBuildVersion>
     <MetadataVersion>61.0.15-preview</MetadataVersion>
     <WDKMetadataVersion>0.12.8-experimental</WDKMetadataVersion>
     <!-- <DiaMetadataVersion>0.2.185-preview-g7e1e6a442c</DiaMetadataVersion> -->
@@ -22,6 +24,7 @@
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="$(CodeAnalysisVersion)" />
     <!-- <PackageVersion Include="Microsoft.Dia.Win32Metadata" Version="0.2.185-preview-g7e1e6a442c" /> -->
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
+    <PackageVersion Include="Microsoft.VisualStudio.Internal.MicroBuild.NonShipping" Version="$(MicroBuildVersion)" />
     <PackageVersion Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageVersion Include="Microsoft.Windows.SDK.Win32Docs" Version="$(ApiDocsVersion)" />
     <PackageVersion Include="Microsoft.Windows.SDK.Win32Metadata" Version="$(MetadataVersion)" />
@@ -34,8 +37,8 @@
     <PackageVersion Include="System.Text.Encodings.Web" Version="4.7.1" />
     <PackageVersion Include="System.Text.Json" Version="4.7.2" />
     <PackageVersion Include="Xunit.Combinatorial" Version="1.6.24" />
-    <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.1" />
-    <PackageVersion Include="xunit" Version="2.8.1" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
+    <PackageVersion Include="xunit" Version="2.9.0" />
   </ItemGroup>
   <ItemGroup Condition="'$(IsTestProject)'!='true'">
     <!-- These versions carefully chosen to support VS 2019 Update 11. -->
@@ -45,8 +48,9 @@
     <PackageVersion Update="System.Runtime.CompilerServices.Unsafe" Version="5.0.0" />
   </ItemGroup>
   <ItemGroup>
-    <GlobalPackageReference Include="CSharpIsNullAnalyzer" Version="0.1.495" />
+    <GlobalPackageReference Include="CSharpIsNullAnalyzer" Version="0.1.593" />
     <GlobalPackageReference Include="DotNetAnalyzers.DocumentationAnalyzers" Version="1.0.0-beta.59" />
+    <GlobalPackageReference Include="Microsoft.VisualStudio.Internal.MicroBuild.VisualStudio" Version="$(MicroBuildVersion)" />
     <GlobalPackageReference Include="Nerdbank.GitVersioning" Version="3.6.139" />
     <GlobalPackageReference Include="Nullable" Version="1.3.1" />
     <GlobalPackageReference Include="StyleCop.Analyzers.Unstable" Version="1.2.0.556" />

diff --git a/azure-pipelines/InsertionMetadataPackage.nuspec b/azure-pipelines/InsertionMetadataPackage.nuspec
diff --git a/azure-pipelines/apiscan.yml b/azure-pipelines/apiscan.yml
@@ -20,7 +20,7 @@ jobs:
     value: $[ dependencies.Windows.outputs['SetPipelineVariables.SymbolsFeatureName'] ]
   - name: NBGV_MajorMinorVersion
     value: $[ dependencies.Windows.outputs['nbgv.NBGV_MajorMinorVersion'] ]
-  - ${{ if parameters.Is1ESPT }}:
+  - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}:
     # https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/25351/APIScan-step-by-step-guide-to-setting-up-a-Pipeline
     - group: VSEng sponsored APIScan # Expected to provide ApiScanClientId
   steps:

diff --git a/azure-pipelines/archive-sourcecode.yml b/azure-pipelines/archive-sourcecode.yml
@@ -36,14 +36,17 @@ extends:
   template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate
   parameters:
     sdl:
-      sourceAnalysisPool: Azure-Pipelines-1ESPT-ExDShared
+      sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES
 
     stages:
     - stage: archive
       jobs:
       - job: archive
         pool:
-          vmImage: ubuntu-20.04
+          name: AzurePipelines-EO
+          demands:
+          - ImageOverride -equals 1ESPT-Ubuntu22.04
+          os: Linux
 
         steps:
         - checkout: self

diff --git a/azure-pipelines/artifacts/VSInsertion.ps1 b/azure-pipelines/artifacts/VSInsertion.ps1
@@ -0,0 +1,35 @@
+# This artifact captures everything needed to insert into VS (NuGet packages, insertion metadata, etc.)
+
+[CmdletBinding()]
+Param (
+)
+
+if ($IsMacOS -or $IsLinux) {
+    # We only package up for insertions on Windows agents since they are where optprof can happen.
+    Write-Verbose "Skipping VSInsertion artifact since we're not on Windows."
+    return @{}
+}
+
+$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..")
+$BuildConfiguration = $env:BUILDCONFIGURATION
+if (!$BuildConfiguration) {
+    $BuildConfiguration = 'Debug'
+}
+
+$NuGetPackages = "$RepoRoot/bin/Packages/$BuildConfiguration/NuGet"
+
+if (!(Test-Path $NuGetPackages)) {
+    Write-Warning "Skipping because NuGet packages haven't been built yet."
+    return @{}
+}
+
+$result = @{
+    "$NuGetPackages" = (Get-ChildItem $NuGetPackages -Recurse)
+}
+
+if ($env:IsOptProf) {
+    $VSRepoPackages = "$RepoRoot/bin/Packages/$BuildConfiguration/VSRepo"
+    $result["$VSRepoPackages"] = (Get-ChildItem "$VSRepoPackages\*.VSInsertionMetadata.*.nupkg");
+}
+
+$result
diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml
@@ -97,8 +97,37 @@ jobs:
   pool: ${{ parameters.windowsPool }}
   variables:
     TestFilter: ""
-  ${{ if parameters.Is1ESPT }}:
+  ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}:
     templateContext:
+      mb:
+        signing:
+          enabled: true
+          zipSources: false
+          ${{ if parameters.RealSign }}:
+            signType: real
+          ${{ else }}:
+            signType: test
+        sbom:
+          enabled: true
+        localization:
+          enabled: ${{ parameters.EnableLocalization }}
+          ${{ if eq(variables['Build.Reason'], 'pullRequest') }}:
+            languages: ENU,JPN
+        optprof:
+          enabled: ${{ parameters.EnableOptProf }}
+          ProfilingInputsDropName: $(ProfilingInputsDropName)
+          OptimizationInputsLookupMethod: DropPrefix
+          DropNamePrefix: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name)
+          ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }}
+          AccessToken: $(System.AccessToken)
+        mbpresteps:
+        - checkout: self
+          fetchDepth: 0 # avoid shallow clone so nbgv can do its work.
+          clean: true
+        - ${{ if parameters.EnableOptProf }}:
+          - powershell: Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(azure-pipelines/variables/ProfilingInputsDropName.ps1)"
+            displayName: ⚙ Set ProfilingInputsDropName for optprof
+
       outputParentDirectory: $(Build.ArtifactStagingDirectory)
       outputs:
       - ${{ each artifact_name in parameters.artifact_names }}:
@@ -108,6 +137,10 @@ jobs:
             targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Windows
             artifactName: ${{ artifact_name }}-Windows
             condition: succeededOrFailed()
+      - output: pipelineArtifact
+        displayName: 📢 Publish VSInsertion-Windows
+        targetPath: $(Build.ArtifactStagingDirectory)/VSInsertion-Windows
+        artifactName: VSInsertion-Windows
       - ${{ if parameters.EnableLocalization }}:
         - output: pipelineArtifact
           displayName: 📢 Publish LocBin-Windows
@@ -123,37 +156,58 @@ jobs:
     - checkout: self
       fetchDepth: 0 # avoid shallow clone so nbgv can do its work.
       clean: true
-      submodules: true
     - ${{ if parameters.EnableOptProf }}:
       - powershell: Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(azure-pipelines/variables/ProfilingInputsDropName.ps1)"
         displayName: ⚙ Set ProfilingInputsDropName for optprof
 
+  - ${{ if eq(variables['Build.Reason'], 'Schedule') }}:
+    - template: schedule-only-steps.yml
+
   - template: install-dependencies.yml
 
   - script: dotnet nbgv cloud -ca
     displayName: ⚙ Set build number
     name: nbgv
 
+  - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}:
+    - template: microbuild.before.yml
+      parameters:
+        EnableLocalization: ${{ parameters.EnableLocalization }}
+        EnableOptProf: ${{ parameters.EnableOptProf }}
+        IsOptProf: ${{ parameters.IsOptProf }}
+        ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }}
+        RealSign: ${{ parameters.RealSign }}
+
   - template: dotnet.yml
     parameters:
       Is1ESPT: ${{ parameters.Is1ESPT }}
       RunTests: ${{ parameters.RunTests }}
       IsOptProf: ${{ parameters.IsOptProf }}
-      RealSign: ${{ parameters.RealSign }}
 
   - ${{ if and(parameters.EnableDotNetFormatCheck, not(parameters.EnableLinuxBuild)) }}:
     - script: dotnet format --verify-no-changes --no-restore
       displayName: 💅 Verify formatted code
 
+  - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}:
+    - template: microbuild.after.yml
+      parameters:
+        EnableOptProf: ${{ parameters.EnableOptProf }}
+        IsOptProf: ${{ parameters.IsOptProf }}
+
 - ${{ if not(parameters.IsOptProf) }}:
   - ${{ if parameters.EnableLinuxBuild }}:
     - job: Linux
       pool: ${{ parameters.linuxPool }}
       variables:
         TestFilter: "&WindowsOnly!=true"
         Platform: NonWindows
-      ${{ if parameters.Is1ESPT }}:
+      ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}:
         templateContext:
+          mb:
+            ${{ if parameters.RealSign }}:
+              signing:
+                enabled: false # enable when building unique artifacts on this agent that must be signed
+                signType: real
           outputParentDirectory: $(Build.ArtifactStagingDirectory)
           outputs:
           - ${{ each artifact_name in parameters.artifact_names }}:
@@ -182,8 +236,13 @@ jobs:
       variables:
         TestFilter: "&WindowsOnly!=true"
         Platform: NonWindows
-      ${{ if parameters.Is1ESPT }}:
+      ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}:
         templateContext:
+          mb:
+            ${{ if parameters.RealSign }}:
+              signing:
+                enabled: false # enable when building unique artifacts on this agent that must be signed
+                signType: real
           outputParentDirectory: $(Build.ArtifactStagingDirectory)
           outputs:
           - ${{ each artifact_name in parameters.artifact_names }}:
@@ -235,5 +294,5 @@ jobs:
         windowsPool: ${{ parameters.windowsPool }}
 
   # Run the integration tests, but only on azure-public so we don't have to worry about multiple pools
-- ${{ if and(parameters.RunTests, not(parameters.Is1ESPT)) }}:
-  - template: integration-test.yml
+  - ${{ if and(parameters.RunTests, not(parameters.Is1ESPT)) }}:
+    - template: integration-test.yml
diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml
@@ -5,32 +5,20 @@ parameters:
   default: false
 - name: Is1ESPT
   type: boolean
-- name: RealSign
-  type: boolean
-  default: false
 
 steps:
 
 - script: dotnet build -t:build,pack --no-restore -c $(BuildConfiguration) -warnaserror /bl:"$(Build.ArtifactStagingDirectory)/build_logs/build.binlog"
   displayName: 🛠 dotnet build
 
-- ${{ if parameters.RealSign }}:
-  - template: ./signstep1.yml
-
-- script: dotnet pack --no-build /p:buildprojectreferences=false -c $(BuildConfiguration) /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/pack.binlog"
-  displayName: 📦 dotnet pack
-
-- ${{ if parameters.RealSign }}:
-  - template: ./signstep2.yml
-
-- powershell: azure-pipelines/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults
-  displayName: 🧪 dotnet test
-  condition: and(succeeded(), ${{ parameters.RunTests }})
+- ${{ if not(parameters.IsOptProf) }}:
+  - powershell: azure-pipelines/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults
+    displayName: 🧪 dotnet test
+    condition: and(succeeded(), ${{ parameters.RunTests }})
 
 - ${{ if parameters.IsOptProf }}:
-  # We have to artifically run this script so that the extra .nupkg is produced for variables/InsertPropsValues.ps1 to notice.
-  - powershell: azure-pipelines\artifacts\VSInsertion.ps1
-    displayName: 🔧 Prepare VSInsertion artifact
+  - script: dotnet pack src\VSInsertionMetadata -c $(BuildConfiguration) -warnaserror /bl:"$(Build.ArtifactStagingDirectory)/build_logs/VSInsertion-Pack.binlog"
+    displayName: 🔧 dotnet pack VSInsertionMetadata
 
 - powershell: azure-pipelines/variables/_pipelines.ps1
   failOnStderr: true