Unify dotnet/runtime on using Arcade -publish

Build.proj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.Build.Traversal">
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(RestoreToolsetOnly)' != 'true'">
     <!-- Subsets are already imported by Directory.Build.props. -->
     <ProjectReference Include="@(ProjectToBuild)" />
     <!-- Only include tasks.proj during restore and build incrementally via a target. -->

Directory.Build.targets

@@ -194,30 +194,4 @@
       <Analyzer Remove="@(_targetingPackExcludedAnalyzerReferenceWithProjectName->Metadata('OriginalIdentity'))" />
     </ItemGroup>
   </Target>
-
-  <!--
-    Arcade SDK versioning is defined by static properties in a targets file: work around this by
-    moving properties based on versioning into a target.
-  -->
-  <Target Name="GetProductVersions">
-    <PropertyGroup>
-      <IncludePreReleaseLabelInPackageVersion Condition="'$(DotNetFinalVersionKind)' != 'release'">true</IncludePreReleaseLabelInPackageVersion>
-      <IncludePreReleaseLabelInPackageVersion Condition="'$(SuppressFinalPackageVersion)' == 'true'">true</IncludePreReleaseLabelInPackageVersion>
-      <IncludePreReleaseLabelInPackageVersion Condition="'$(IsShipping)' != 'true'">true</IncludePreReleaseLabelInPackageVersion>
-
-      <IncludeBuildNumberInPackageVersion Condition="'$(StabilizePackageVersion)' != 'true'">true</IncludeBuildNumberInPackageVersion>
-      <IncludeBuildNumberInPackageVersion Condition="'$(SuppressFinalPackageVersion)' == 'true'">true</IncludeBuildNumberInPackageVersion>
-      <IncludeBuildNumberInPackageVersion Condition="'$(IsShipping)' != 'true'">true</IncludeBuildNumberInPackageVersion>
-
-      <ProductVersionSuffix Condition="'$(IncludePreReleaseLabelInPackageVersion)' == 'true'">-$(VersionSuffix)</ProductVersionSuffix>
-      <ProductBandVersion Condition="'$(ProductBandVersion)' == ''">$(MajorVersion).$(MinorVersion)</ProductBandVersion>
-      <ProductionVersion Condition="'$(ProductionVersion)' == ''">$(ProductBandVersion).$(PatchVersion)</ProductionVersion>
-      <ProductVersion>$(ProductionVersion)$(ProductVersionSuffix)</ProductVersion>
-
-      <SharedFrameworkNugetVersion>$(ProductVersion)</SharedFrameworkNugetVersion>
-      <NuGetVersion>$(SharedFrameworkNugetVersion)</NuGetVersion>
-      <InstallersRelativePath>Runtime/$(SharedFrameworkNugetVersion)/</InstallersRelativePath>
-    </PropertyGroup>
-  </Target>
-
 </Project>

eng/Publishing.props

@@ -1,39 +1,206 @@
-<Project>
+<Project InitialTargets="ValidateDownloadedAssets">
 
   <PropertyGroup>
     <ProducesDotNetReleaseShippingAssets>true</ProducesDotNetReleaseShippingAssets>
+    <!-- This avoids creating VS.*.symbols.nupkg packages that are identical to the original package. -->
+    <AutoGenerateSymbolPackages>false</AutoGenerateSymbolPackages>
+    <!-- Set PlatformName to TargetArchitecture to create unique build manifest files. -->
+    <PlatformName Condition="'$(TargetArchitecture)' != ''">$(TargetArchitecture)</PlatformName>
   </PropertyGroup>
 
-  <!-- Include installer archives and packages which aren't globbed by default.
-       Don't include Symbols archive as it is already included in Arcade's Publish.proj, with correct blob path. -->
-  <Target Name="PublishRuntimeInstallers"
-          BeforeTargets="BeforePublish"
-          Condition="'$(DotNetBuildRepo)' == 'true'">
-    <!-- Retrieve runtime's runtime pack product version.
+  <!--
+    Look through the downloaded artifacts to figure out how to ship them. Creates item groups for
+    other types of publishing to use.
+
+    DownloadDirectory is expected to have some directory inside, which then contains a dir for each
+    build job name. For example, this nupkg would be found:
+
+      $(DownloadDirectory)IntermediateArtifacts\windows_x64\Shipping\a.nupkg
+  -->
+  <ItemGroup Condition="'$(DotNetBuildRepo)' != 'true'">
+    <DownloadedArtifactFile Include="$(DownloadDirectory)**" />
+    <DownloadedSymbolNupkgFile Include="$(DownloadDirectory)**\*.symbols.nupkg" />
+    <DownloadedWixPdbFile Include="$(DownloadDirectory)**\*.wixpdb" />
+    <DownloadedWixpackFile Include="$(DownloadDirectory)**\*.wixpack.zip" Condition="'$(PostBuildSign)' != 'true'" />
+    <DownloadedWorkloadsVSInsertionFile Include="$(DownloadDirectory)*\workloads-vs\**\*" />
+    <DownloadedNupkgFile Include="$(DownloadDirectory)**\*.nupkg" Exclude="@(DownloadedSymbolNupkgFile)" />
+
+    <!-- Exclude symbol packages from have a NuGet signature. These are never pushed to NuGet.org or
+          other feeds (in fact, that have identical identity to their non-symbol variant) -->
+    <DownloadedSymbolPackagesWithoutPaths Include="@(DownloadedSymbolNupkgFile->'%(Filename)%(Extension)')" />
+    <FileSignInfo Include="@(DownloadedSymbolPackagesWithoutPaths->Distinct())" CertificateName="None" />
+
+    <!-- Add files that are not affected by filtering and create checksum for them. -->
+    <UploadToBlobStorageFile
+      Include="@(DownloadedArtifactFile)"
+      Exclude="@(DownloadedSymbolNupkgFile);
+                @(DownloadedNupkgFile);
+                @(DownloadedWixPdbFile);
+                @(DownloadedWorkloadsVSInsertionFile);
+                @(DownloadedWixpackFile)" />
+
+    <!--
+      Filter out the RID-specific (Runtime) nupkgs and RID-agnostic nupkgs. RID-specific packages
+      are published from every job. RID-agnostic nupkgs are built with the same ID/version by
+      every job, so one specific job's outputs must be picked to sign and publish.
+    -->
+
+    <!-- RID-specific framework packs. -->
+    <RuntimeNupkgFile
+      Include="
+        $(DownloadDirectory)**\Microsoft.*.Runtime.*.nupkg;
+        $(DownloadDirectory)**\Microsoft.*.App.Host.*.nupkg;
+        $(DownloadDirectory)**\Microsoft.*.App.Crossgen2.*.nupkg"
+      Exclude="@(DownloadedSymbolNupkgFile)" />
+
+    <!-- VS insertion packages, carrying RID-specific installers. -->
+    <RuntimeNupkgFile
+      Include="$(DownloadDirectory)**\VS.Redist.Common.*.nupkg"
+      Exclude="@(DownloadedSymbolNupkgFile)" />
+
+    <!--
+      Workloads VS insertion artifacts produced by src/workloads/workloads.csproj. Only grab
+      the zip artifacts as they're grouped by SDK feature band which correlates with specific VS versions.
+    -->
+    <WorkloadsVSInsertionFile Include="$(DownloadDirectory)*\workloads-vs\**\*.zip" />
+
+    <!--
+      Runtime packages associated with some identity packages. Need to exclude "runtime.native.*"
+      because Libraries produces some "runtime.native.Foo" packages with
+      "runtime.<rid>.runtime.native.Foo" identity packages.
+    -->
+    <RuntimeNupkgFile
+      Include="$(DownloadDirectory)**\runtime.*.nupkg"
+      Exclude="
+        $(DownloadDirectory)**\runtime.native.*.nupkg;
+        @(DownloadedSymbolNupkgFile)" />
+
+    <!--
+      Packages that aren't matched above as RID-specific are considered RID-agnostic. Also include
+      the AllConfigurations packages from the Libraries build.
+    -->
+    <RidAgnosticNupkgToPublishFile
+      Include="
+        $(DownloadDirectory)**\Microsoft.NET.Workload.Mono.Toolchain.*Manifest-*.nupkg;
+        $(DownloadDirectory)**\Microsoft.NET.Sdk.WebAssembly.Pack.*.nupkg;
+        $(DownloadDirectory)*\$(PublishRidAgnosticPackagesFromPlatform)\**\*.nupkg;
+        $(DownloadDirectory)*\*AllConfigurations\**\*.nupkg"
+      Exclude="@(RuntimeNupkgFile);@(DownloadedSymbolNupkgFile)" />
+
+    <TransportPackagesToPublishFile
+      Include="$(DownloadDirectory)**\*Transport*.nupkg"
+      Exclude="@(RuntimeNupkgFile);@(RidAgnosticNupkgToPublishFile);@(DownloadedSymbolNupkgFile)" />
+
+    <NupkgToPublishFile Include="@(RuntimeNupkgFile);@(RidAgnosticNupkgToPublishFile);@(TransportPackagesToPublishFile)" />
+
+    <!--
+      Assuming all symbol packages ship and can be found by turning .nupkg => .symbols.nupkg, find
+      them. Don't check for missing symbol packages here: some nupkgs don't have them for valid
+      reasons, such as the VS insertion packages that transport MSIs. Symbol package validation
+      will check for symbol completeness with file-by-file granularity rather than looking for
+      missing symbols.nupkg files: https://github.com/dotnet/arcade/issues/2499.
+
+      Handles several conventions:
+      * NonShipping packages have symbol nupkgs that are Shipping.
+      * Shipping packages have symbol packages in a "symbols" subdirectory.
+    -->
+    <PotentialSymbolNupkgToPublishFile
+      Include="
+        @(NupkgToPublishFile->Replace('\NonShipping\', '\Shipping\')->Replace('.nupkg', '.symbols.nupkg'));
+        @(NupkgToPublishFile->Replace('\NonShipping\', '\NonShipping\symbols\')->Replace('.nupkg', '.symbols.nupkg'));
+        @(NupkgToPublishFile->Replace('\Shipping\', '\Shipping\symbols\')->Replace('.nupkg', '.symbols.nupkg'))" />
+
+    <SymbolNupkgToPublishFile Include="@(PotentialSymbolNupkgToPublishFile->Distinct()->Exists())" />
+
+    <!-- Packages -->
+    <Artifact Include="@(NupkgToPublishFile)"
+              IsShipping="$([System.String]::new('%(Identity)').Contains('\Shipping\'))"
+              PublishFlatContainer="false" />
+    <Artifact Include="@(SymbolNupkgToPublishFile)" PublishFlatContainer="false" />
+
+    <!-- Blob storage -->
+    <Artifact Include="@(UploadToBlobStorageFile)"
+              Exclude="@(NupkgToPublishFile);@(SymbolNupkgToPublishFile)"
+              IsShipping="$([System.String]::new('%(Identity)').Contains('\Shipping\'))"
+              ChecksumPath="%(FullPath).sha512" />
+    <Artifact Include="@(WorkloadsVSInsertionFile)"
+              SubBlobFolder="workloads/"
+              IsShipping="$([System.String]::new('%(Identity)').Contains('\Shipping\'))" />
+  </ItemGroup>
+
+  <Target Name="GetNonStableProductVersion">
+    <!-- Retrieve the non-stable runtime pack product version.
          Don't stabilize the package version in order to retrieve the VersionSuffix. -->
     <MSBuild Projects="$(RepoRoot)src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj"
              Targets="ReturnProductVersion"
              Properties="IsShipping=false;
                          Crossgen2SdkOverridePropsPath=;
                          Crossgen2SdkOverrideTargetsPath=">
-      <Output TaskParameter="TargetOutputs" PropertyName="RuntimeRuntimePackProductVersion" />
+      <Output TaskParameter="TargetOutputs" PropertyName="NonStableProductVersion" />
+    </MSBuild>
+  </Target>
+
+  <!--
+    Only generate the productVersion.txt and runtime-productVersion.txt files when we're building
+    either the full VMR repo or not building in the VMR infrastructure.
+    This ensures that we don't produce these files in the "Repo source build" builds,
+    but we do produce them in both the VMR and the runtime official build.
+  -->
+  <PropertyGroup>
+    <_ShouldGenerateProductVersionFiles Condition="'$(DotNetBuildRepo)' == 'true' and '$(DotNetBuildOrchestrator)' == 'true'">true</_ShouldGenerateProductVersionFiles>
+    <_ShouldGenerateProductVersionFiles Condition="'$(DotNetBuildRepo)' != 'true' and '$(DotNetBuildOrchestrator)' != 'true'">true</_ShouldGenerateProductVersionFiles>
+  </PropertyGroup>
+
+  <Target Name="GenerateProductVersionFiles"
+          DependsOnTargets="GetNonStableProductVersion"
+          BeforeTargets="PublishToAzureDevOpsArtifacts"
+          Condition="'$(_ShouldGenerateProductVersionFiles)' == 'true'">
+    <!-- Retrieve the runtime pack product version. -->
+    <MSBuild Projects="$(RepoRoot)src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj"
+             Targets="ReturnProductVersion"
+             Properties="Crossgen2SdkOverridePropsPath=;
+                         Crossgen2SdkOverrideTargetsPath=">
+      <Output TaskParameter="TargetOutputs" PropertyName="ProductVersionForVersionsFile" />
     </MSBuild>
 
     <ItemGroup>
-      <InstallerToPublish Include="$(ArtifactsPackagesDir)**\*.tar.gz;
-                                   $(ArtifactsPackagesDir)**\*.zip;
-                                   $(ArtifactsPackagesDir)**\*.deb;
-                                   $(ArtifactsPackagesDir)**\*.rpm;
-                                   $(ArtifactsPackagesDir)**\*.pkg;
-                                   $(ArtifactsPackagesDir)**\*.exe;
-                                   $(ArtifactsPackagesDir)**\*.msi"
-                          Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" />
-
-      <ItemsToPushToBlobFeed Include="@(InstallerToPublish)"
-                             IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))"
-                             PublishFlatContainer="true"
-                             RelativeBlobPath="Runtime/$(RuntimeRuntimePackProductVersion)/%(Filename)%(Extension)" />
+      <ProductVersionFile Include="$(ArtifactsShippingPackagesDir)productVersion.txt" />
+      <ProductVersionFile Include="$(ArtifactsShippingPackagesDir)runtime-productVersion.txt" />
     </ItemGroup>
+
+    <!-- Generate productVersion.txt file containing the product version. -->
+    <WriteLinesToFile File="%(ProductVersionFile.Identity)"
+                      Lines="$(ProductVersionForVersionsFile)"
+                      Overwrite="true"
+                      Encoding="ASCII" />
+
+    <ItemGroup>
+      <Artifact Include="@(ProductVersionFile)"
+                RelativeBlobPath="Runtime/$(NonStableProductVersion)/%(Filename)%(Extension)" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="AddRelativeBlobPathToInstallerArtifacts"
+          DependsOnTargets="GetNonStableProductVersion"
+          BeforeTargets="PublishToAzureDevOpsArtifacts"
+          AfterTargets="GenerateChecksumsFromArtifacts">
+    <ItemGroup>
+      <Artifact Condition="'%(Artifact.PublishFlatContainer)' == 'true' and '%(Artifact.RelativeBlobPath)' == ''"
+                RelativeBlobPath="Runtime/$(NonStableProductVersion)/%(Artifact.SubBlobFolder)%(Filename)%(Extension)" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="ValidateDownloadedAssets" Condition="'$(DotNetBuildRepo)' != 'true'">
+    <Error Text="DownloadDirectory is not set." Condition="'$(DownloadDirectory)' == ''" />
+    <Error Condition="'@(SymbolNupkgToPublishFile)' == ''" Text="No symbol packages found." />
+
+    <!--
+      Duplicate RuntimeNupkgFile items mean artifact upload will fail, but only after another hour
+      of signing. Detect this early. It's possible to automatically "fix" this with Distinct(),
+      however the patterns should be fairly specific: this is likely a build infra mistake that
+      should be corrected.
+    -->
+    <Error Text="Duplicate RuntimeNupkgFile entries for: %(RuntimeNupkgFile.Identity)" Condition="@(RuntimeNupkgFile->Count()) &gt; 1" />
   </Target>
 
-</Project>
+</Project>

eng/Subsets.props

@@ -206,7 +206,6 @@
     <SubsetName Include="Packs.Tests" Description="The framework pack tests." />
 
     <!-- Utility -->
-    <SubsetName Include="publish" OnDemand="true" Description="Generate asset manifests and prepare to publish to BAR." />
     <SubsetName Include="RegenerateDownloadTable" OnDemand="true" Description="Regenerates the nightly build download table" />
     <SubsetName Include="RegenerateThirdPartyNotices" OnDemand="true" Description="Regenerates the THIRD-PARTY-NOTICES.TXT file based on other repos' TPN files." />
     <SubsetName Include="tasks" OnDemand="true" Description="Build the repo local task projects." />
@@ -560,10 +559,6 @@
     <ProjectToBuild Include="@(TestProjectToBuild)" BuildInParallel="true" Test="true" Category="packs" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subset.Contains('+publish+'))">
-    <ProjectToBuild Include="$(InstallerProjectRoot)prepare-artifacts.proj" Pack="true" Category="publish" />
-  </ItemGroup>
-
   <!-- Utility -->
   <ItemGroup Condition="$(_subset.Contains('+regeneratedownloadtable+'))">
     <ProjectToBuild Include="$(RepositoryEngineeringDir)regenerate-download-table.proj" Pack="true" />

eng/pipelines/official/jobs/prepare-signed-artifacts.yml

@@ -43,13 +43,14 @@ jobs:
 
     steps:
     - script: >-
-        build.cmd -ci
-        -subset publish
-        -configuration Release
+        build.cmd -restore -sign -publish -ci -configuration Release
+        /p:RestoreToolsetOnly=true
         /p:PublishRidAgnosticPackagesFromPlatform=${{ parameters.PublishRidAgnosticPackagesFromPlatform }}
+        /p:DownloadDirectory=$(Build.SourcesDirectory)\artifacts\PackageDownload\
         /p:OfficialBuildId=$(Build.BuildNumber)
         /p:SignType=$(_SignType)
         /p:DotNetSignType=$(_SignType)
+        /p:DotNetPublishUsingPipelines=true
         /bl:$(Build.SourcesDirectory)\prepare-artifacts.binlog
       displayName: Prepare artifacts and upload to build
 

src/installer/Directory.Build.props

@@ -14,11 +14,4 @@
       want to ensure we use the correct packages. -->
     <DisableImplicitNuGetFallbackFolder>true</DisableImplicitNuGetFallbackFolder>
   </PropertyGroup>
-
-  <!-- Construct the path to the sign project, so that signing can be called during prepare-artifacts -->
-  <PropertyGroup>
-    <ArcadeSdkMSBuildProjectDir>$([System.IO.Path]::GetDirectoryName('$(ArcadeSdkBuildTasksAssembly)'))\..\</ArcadeSdkMSBuildProjectDir>
-    <ArcadeSdkSignProject>$(ArcadeSdkMSBuildProjectDir)Sign.proj</ArcadeSdkSignProject>
-  </PropertyGroup>
-
 </Project>

src/installer/pkg/projects/Directory.Build.targets

@@ -5,7 +5,9 @@
       If we are stabilizing set the StableVersion property for the packages.
       Needs to happen in Directory.Build.targets to allow all the pkgproj's to set Version property first.
     -->
-    <StableVersion Condition="'$(IncludePreReleaseLabelInPackageVersion)' != 'true'">$(Version)</StableVersion>
+    <StableVersion Condition="'$(DotNetFinalVersionKind)' == 'release' and
+                              '$(SuppressFinalPackageVersion)' != 'true' and
+                              '$(IsShipping)' == 'true'">$(Version)</StableVersion>
 
     <!--
       Explicitly re-set the symbol package output path. The pkgproj files here end up importing the targets from
@@ -16,20 +18,6 @@
     <SymbolPackageOutputPath>$(PackageOutputPath)</SymbolPackageOutputPath>
   </PropertyGroup>
 
-  <Target Name="SetTargetBasedPackageVersion"
-          BeforeTargets="GenerateNuSpec"
-          DependsOnTargets="GetProductVersions">
-    <PropertyGroup>
-      <Version>$(ProductVersion)</Version>
-      <!--
-        PackageVersion is normally calculated using Version during static property evaluation, but
-        we need some info from GetProductVersions, so it's too late to rely on that. We need to set
-        both in target evaluation, here.
-      -->
-      <PackageVersion>$(ProductVersion)</PackageVersion>
-    </PropertyGroup>
-  </Target>
-
   <!--
     Remove duplicate files returned by restore. The resolve task performs extra detection to pick up
     a PDB file for any file listed in the assets file. This causes duplicates if the assets file