Include versions and checksums in VMR publishing and remove prepare-artifacts.proj

Directory.Build.targets

@@ -201,29 +201,4 @@
     </ItemGroup>
   </Target>
 
-  <!--
-    Arcade SDK versioning is defined by static properties in a targets file: work around this by
-    moving properties based on versioning into a target.
-  -->
-  <Target Name="GetProductVersions">
-    <PropertyGroup>
-      <IncludePreReleaseLabelInPackageVersion Condition="'$(DotNetFinalVersionKind)' != 'release'">true</IncludePreReleaseLabelInPackageVersion>
-      <IncludePreReleaseLabelInPackageVersion Condition="'$(SuppressFinalPackageVersion)' == 'true'">true</IncludePreReleaseLabelInPackageVersion>
-      <IncludePreReleaseLabelInPackageVersion Condition="'$(IsShipping)' != 'true'">true</IncludePreReleaseLabelInPackageVersion>
-
-      <IncludeBuildNumberInPackageVersion Condition="'$(StabilizePackageVersion)' != 'true'">true</IncludeBuildNumberInPackageVersion>
-      <IncludeBuildNumberInPackageVersion Condition="'$(SuppressFinalPackageVersion)' == 'true'">true</IncludeBuildNumberInPackageVersion>
-      <IncludeBuildNumberInPackageVersion Condition="'$(IsShipping)' != 'true'">true</IncludeBuildNumberInPackageVersion>
-
-      <ProductVersionSuffix Condition="'$(IncludePreReleaseLabelInPackageVersion)' == 'true'">-$(VersionSuffix)</ProductVersionSuffix>
-      <ProductBandVersion Condition="'$(ProductBandVersion)' == ''">$(MajorVersion).$(MinorVersion)</ProductBandVersion>
-      <ProductionVersion Condition="'$(ProductionVersion)' == ''">$(ProductBandVersion).$(PatchVersion)</ProductionVersion>
-      <ProductVersion>$(ProductionVersion)$(ProductVersionSuffix)</ProductVersion>
-
-      <SharedFrameworkNugetVersion>$(ProductVersion)</SharedFrameworkNugetVersion>
-      <NuGetVersion>$(SharedFrameworkNugetVersion)</NuGetVersion>
-      <InstallersRelativePath>Runtime/$(SharedFrameworkNugetVersion)/</InstallersRelativePath>
-    </PropertyGroup>
-  </Target>
-
 </Project>

eng/Publishing.props

@@ -1,39 +1,193 @@
-<Project>
+<Project InitialTargets="ValidateDownloadedAssets">
 
   <PropertyGroup>
     <ProducesDotNetReleaseShippingAssets>true</ProducesDotNetReleaseShippingAssets>
+    <!-- This avoids creating VS.*.symbols.nupkg packages that are identical to the original package. -->
+    <AutoGenerateSymbolPackages>false</AutoGenerateSymbolPackages>
+    <!-- Set PlatformName to TargetArchitecture to create unique build manifest files. -->
+    <PlatformName Condition="'$(TargetArchitecture)' != ''">$(TargetArchitecture)</PlatformName>
   </PropertyGroup>
 
-  <!-- Include installer archives and packages which aren't globbed by default.
-       Don't include Symbols archive as it is already included in Arcade's Publish.proj, with correct blob path. -->
-  <Target Name="PublishRuntimeInstallers"
-          BeforeTargets="BeforePublish"
-          Condition="'$(DotNetBuildRepo)' == 'true'">
-    <!-- Retrieve runtime's runtime pack product version.
+  <!--
+    Look through the downloaded artifacts to figure out how to ship them. Creates item groups for
+    other types of publishing to use.
+
+    DownloadDirectory is expected to have some directory inside, which then contains a dir for each
+    build job name. For example, this nupkg would be found:
+
+      $(DownloadDirectory)IntermediateArtifacts\windows_x64\Shipping\a.nupkg
+  -->
+  <ItemGroup Condition="'$(SignAndPublishAsJoinPoint)' == 'true'">
+    <DownloadedArtifactFile Include="$(DownloadDirectory)**" />
+    <DownloadedSymbolNupkgFile Include="$(DownloadDirectory)**\*.symbols.nupkg" />
+    <DownloadedWixPdbFile Include="$(DownloadDirectory)**\*.wixpdb" />
+    <DownloadedWixpackFile Include="$(DownloadDirectory)**\*.wixpack.zip" Condition="'$(PostBuildSign)' != 'true'" />
+    <DownloadedWorkloadsVSInsertionFile Include="$(DownloadDirectory)*\workloads-vs\**\*" />
+    <DownloadedNupkgFile Include="$(DownloadDirectory)**\*.nupkg" Exclude="@(DownloadedSymbolNupkgFile)" />
+
+    <!-- Exclude symbol packages from have a NuGet signature. These are never pushed to NuGet.org or
+          other feeds (in fact, that have identical identity to their non-symbol variant) -->
+    <DownloadedSymbolPackagesWithoutPaths Include="@(DownloadedSymbolNupkgFile->'%(Filename)%(Extension)')" />
+    <FileSignInfo Include="@(DownloadedSymbolPackagesWithoutPaths->Distinct())" CertificateName="None" />
+
+    <!-- Add files that are not affected by filtering and create checksum for them. -->
+    <UploadToBlobStorageFile
+      Include="@(DownloadedArtifactFile)"
+      Exclude="@(DownloadedSymbolNupkgFile);
+                @(DownloadedNupkgFile);
+                @(DownloadedWixPdbFile);
+                @(DownloadedWorkloadsVSInsertionFile);
+                @(DownloadedWixpackFile)" />
+
+    <!--
+      Filter out the RID-specific (Runtime) nupkgs and RID-agnostic nupkgs. RID-specific packages
+      are published from every job. RID-agnostic nupkgs are built with the same ID/version by
+      every job, so one specific job's outputs must be picked to sign and publish.
+    -->
+
+    <!-- RID-specific framework packs. -->
+    <RuntimeNupkgFile
+      Include="
+        $(DownloadDirectory)**\Microsoft.*.Runtime.*.nupkg;
+        $(DownloadDirectory)**\Microsoft.*.App.Host.*.nupkg;
+        $(DownloadDirectory)**\Microsoft.*.App.Crossgen2.*.nupkg"
+      Exclude="@(DownloadedSymbolNupkgFile)" />
+
+    <!-- VS insertion packages, carrying RID-specific installers. -->
+    <RuntimeNupkgFile
+      Include="$(DownloadDirectory)**\VS.Redist.Common.*.nupkg"
+      Exclude="@(DownloadedSymbolNupkgFile)" />
+
+    <!--
+      Workloads VS insertion artifacts produced by src/workloads/workloads.csproj. Only grab
+      the zip artifacts as they're grouped by SDK feature band which correlates with specific VS versions.
+    -->
+    <WorkloadsVSInsertionFile Include="$(DownloadDirectory)*\workloads-vs\**\*.zip" />
+
+    <!--
+      Runtime packages associated with some identity packages. Need to exclude "runtime.native.*"
+      because Libraries produces some "runtime.native.Foo" packages with
+      "runtime.<rid>.runtime.native.Foo" identity packages.
+    -->
+    <RuntimeNupkgFile
+      Include="$(DownloadDirectory)**\runtime.*.nupkg"
+      Exclude="
+        $(DownloadDirectory)**\runtime.native.*.nupkg;
+        @(DownloadedSymbolNupkgFile)" />
+
+    <!--
+      Packages that aren't matched above as RID-specific are considered RID-agnostic. Also include
+      the AllConfigurations packages from the Libraries build.
+    -->
+    <RidAgnosticNupkgToPublishFile
+      Include="
+        $(DownloadDirectory)**\Microsoft.NET.Workload.Mono.Toolchain.*Manifest-*.nupkg;
+        $(DownloadDirectory)**\Microsoft.NET.Sdk.WebAssembly.Pack.*.nupkg;
+        $(DownloadDirectory)*\$(PublishRidAgnosticPackagesFromPlatform)\**\*.nupkg;
+        $(DownloadDirectory)*\*AllConfigurations\**\*.nupkg"
+      Exclude="@(RuntimeNupkgFile);@(DownloadedSymbolNupkgFile)" />
+
+    <TransportPackagesToPublishFile
+      Include="$(DownloadDirectory)**\*Transport*.nupkg"
+      Exclude="@(RuntimeNupkgFile);@(RidAgnosticNupkgToPublishFile);@(DownloadedSymbolNupkgFile)" />
+
+    <NupkgToPublishFile Include="@(RuntimeNupkgFile);@(RidAgnosticNupkgToPublishFile);@(TransportPackagesToPublishFile)" />
+
+    <!--
+      Assuming all symbol packages ship and can be found by turning .nupkg => .symbols.nupkg, find
+      them. Don't check for missing symbol packages here: some nupkgs don't have them for valid
+      reasons, such as the VS insertion packages that transport MSIs. Symbol package validation
+      will check for symbol completeness with file-by-file granularity rather than looking for
+      missing symbols.nupkg files: https://github.com/dotnet/arcade/issues/2499.
+
+      Handles several conventions:
+      * NonShipping packages have symbol nupkgs that are Shipping.
+      * Shipping packages have symbol packages in a "symbols" subdirectory.
+    -->
+    <PotentialSymbolNupkgToPublishFile
+      Include="
+        @(NupkgToPublishFile->Replace('\NonShipping\', '\Shipping\')->Replace('.nupkg', '.symbols.nupkg'));
+        @(NupkgToPublishFile->Replace('\NonShipping\', '\NonShipping\symbols\')->Replace('.nupkg', '.symbols.nupkg'));
+        @(NupkgToPublishFile->Replace('\Shipping\', '\Shipping\symbols\')->Replace('.nupkg', '.symbols.nupkg'))" />
+
+    <SymbolNupkgToPublishFile Include="@(PotentialSymbolNupkgToPublishFile->Distinct())" Condition="Exists('%(Identity)')" />
+
+    <!-- Packages -->
+    <Artifact Include="@(NupkgToPublishFile)"
+              IsShipping="$([System.String]::new('%(Identity)').Contains('\Shipping\'))"
+              PublishFlatContainer="false" />
+    <Artifact Include="@(SymbolNupkgToPublishFile)" PublishFlatContainer="false" />
+
+    <!-- Blob storage -->
+    <Artifact Include="@(UploadToBlobStorageFile)"
+              Exclude="@(NupkgToPublishFile);@(SymbolNupkgToPublishFile)"
+              IsShipping="$([System.String]::new('%(Identity)').Contains('\Shipping\'))"
+              ChecksumPath="%(FullPath).sha512" />
+    <Artifact Include="@(WorkloadsVSInsertionFile)"
+              SubBlobFolder="workloads/"
+              IsShipping="$([System.String]::new('%(Identity)').Contains('\Shipping\'))" />
+  </ItemGroup>
+
+  <Target Name="GetNonStableProductVersion">
+    <!-- Retrieve the non-stable runtime pack product version.
          Don't stabilize the package version in order to retrieve the VersionSuffix. -->
     <MSBuild Projects="$(RepoRoot)src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj"
              Targets="ReturnProductVersion"
              Properties="IsShipping=false;
                          Crossgen2SdkOverridePropsPath=;
                          Crossgen2SdkOverrideTargetsPath=">
-      <Output TaskParameter="TargetOutputs" PropertyName="RuntimeRuntimePackProductVersion" />
+      <Output TaskParameter="TargetOutputs" PropertyName="NonStableProductVersion" />
+    </MSBuild>
+  </Target>
+
+  <Target Name="GenerateProductVersionFiles"
+          DependsOnTargets="GetNonStableProductVersion"
+          BeforeTargets="PublishToAzureDevOpsArtifacts">
+    <!-- Retrieve the runtime pack product version. -->
+    <MSBuild Projects="$(RepoRoot)src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj"
+             Targets="ReturnProductVersion"
+             Properties="Crossgen2SdkOverridePropsPath=;
+                         Crossgen2SdkOverrideTargetsPath=">
+      <Output TaskParameter="TargetOutputs" PropertyName="ProductVersionForVersionsFile" />
     </MSBuild>
 
     <ItemGroup>
-      <InstallerToPublish Include="$(ArtifactsPackagesDir)**\*.tar.gz;
-                                   $(ArtifactsPackagesDir)**\*.zip;
-                                   $(ArtifactsPackagesDir)**\*.deb;
-                                   $(ArtifactsPackagesDir)**\*.rpm;
-                                   $(ArtifactsPackagesDir)**\*.pkg;
-                                   $(ArtifactsPackagesDir)**\*.exe;
-                                   $(ArtifactsPackagesDir)**\*.msi"
-                          Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" />
-
-      <ItemsToPushToBlobFeed Include="@(InstallerToPublish)"
-                             IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))"
-                             PublishFlatContainer="true"
-                             RelativeBlobPath="Runtime/$(RuntimeRuntimePackProductVersion)/%(Filename)%(Extension)" />
+      <ProductVersionFile Include="$(ArtifactsShippingPackagesDir)productVersion.txt" />
+      <ProductVersionFile Include="$(ArtifactsShippingPackagesDir)runtime-productVersion.txt" />
     </ItemGroup>
+
+    <!-- Generate productVersion.txt file containing the product version. -->
+    <WriteLinesToFile File="%(ProductVersionFile.Identity)"
+                      Lines="$(ProductVersionForVersionsFile)"
+                      Overwrite="true"
+                      Encoding="ASCII" />
+
+    <ItemGroup>
+      <Artifact Include="@(ProductVersionFile)"
+                RelativeBlobPath="Runtime/$(NonStableProductVersion)/%(Filename)%(Extension)" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="AddRelativeBlobPathToInstallerArtifacts"
+          DependsOnTargets="GetNonStableProductVersion"
+          BeforeTargets="PublishToAzureDevOpsArtifacts"
+          AfterTargets="GenerateChecksumsFromArtifacts">
+    <ItemGroup>
+      <Artifact Condition="'%(Artifact.PublishFlatContainer)' == 'true' and '%(Artifact.RelativeBlobPath)' == ''"
+                RelativeBlobPath="Runtime/$(NonStableProductVersion)/%(Artifact.SubBlobFolder)%(Filename)%(Extension)" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="ValidateDownloadedAssets" Condition="'$(SignAndPublishAsJoinPoint)' == 'true'">
+    <Error Condition="'@(SymbolNupkgToPublishFile)' == ''" Text="No symbol packages found." />
+
+    <!--
+      Duplicate RuntimeNupkgFile items mean artifact upload will fail, but only after another hour
+      of signing. Detect this early. It's possible to automatically "fix" this with Distinct(),
+      however the patterns should be fairly specific: this is likely a build infra mistake that
+      should be corrected.
+    -->
+    <Error Text="Duplicate RuntimeNupkgFile entries for: %(RuntimeNupkgFile.Identity)" Condition="@(RuntimeNupkgFile->Count()) &gt; 1" />
   </Target>
 
 </Project>

eng/Signing.props

@@ -1,21 +1,16 @@
 <Project>
-  <PropertyGroup>
-    <!--
-      Windows arm/arm64 jobs don't have MSIs to sign. Keep it simple: allow not finding any matches
-      here and rely on overall signing validation.
 
-      During post build signing, there are no packages to sign during SignFinalPackages.
-    -->
-    <AllowEmptySignList>true</AllowEmptySignList>
+  <PropertyGroup>
+    <!-- Runtime needs to sign all artifacts (including other verticals) on Windows until a cross-plat signing solution
+         is available that has all the required features. Inside the VMR we publish every vertical which is why this
+         property is then not true. -->
+    <SignAndPublishAsJoinPoint Condition="'$(DownloadDirectory)' != ''">true</SignAndPublishAsJoinPoint>
+    <!-- When signing and publishing as a join point, don't glob for default artifacts. Instead, artifacts are
+         manually added from a provided DownloadDirectory property. -->
+    <EnableDefaultArtifacts Condition="'$(SignAndPublishAsJoinPoint)' == 'true'">false</EnableDefaultArtifacts>
   </PropertyGroup>
 
   <ItemGroup>
-    <!--
-      Replace the default items to sign with the specific set we want. This allows the build to call
-      Arcade's Sign.proj multiple times for different sets of files as the build progresses.
-    -->
-    <ItemsToSign Remove="@(ItemsToSign)" />
-
     <!-- apphost and comhost template files are not signed, by design. -->
     <FileSignInfo Include="apphost.exe;singlefilehost.exe;comhost.dll" CertificateName="None" />
 
@@ -37,12 +32,6 @@
     <FileSignInfo Include="Mono.Cecil.Pdb.dll" CertificateName="3PartySHA2" />
     <FileSignInfo Include="Mono.Cecil.Rocks.dll" CertificateName="3PartySHA2" />
 
-    <!-- Exclude symbol packages from have a NuGet signature. These are never pushed to NuGet.org or
-         other feeds (in fact, that have identical identity to their non-symbol variant) -->
-    <DownloadedSymbolPackages Include="$(DownloadDirectory)**\*.symbols.nupkg" />
-    <DownloadedSymbolPackagesWithoutPaths Include="@(DownloadedSymbolPackages->'%(Filename)%(Extension)')" />
-    <FileSignInfo Include="@(DownloadedSymbolPackagesWithoutPaths->Distinct())" CertificateName="None" />
-
     <FileExtensionSignInfo Include=".msi" CertificateName="MicrosoftDotNet500" />
     <FileExtensionSignInfo Include=".pkg" CertificateName="8003" />
     <FileExtensionSignInfo Include=".deb;.rpm" CertificateName="LinuxSign" />
@@ -55,14 +44,30 @@
     <FileSignInfo Update="@(FileSignInfo->WithMetadataValue('CertificateName','Microsoft400'))" CertificateName="MicrosoftDotNet500" />
   </ItemGroup>
 
-  <ItemGroup Condition="'$(PrepareArtifacts)' == 'true'">
-    <ItemsToSignWithPaths Include="$(DownloadDirectory)**\*.msi" />
-    <ItemsToSignWithPaths Include="$(DownloadDirectory)**\*.exe" />
-    <ItemsToSignWithPaths Include="$(DownloadDirectory)**\*.nupkg" />
-    <ItemsToSignWithPaths Include="$(DownloadDirectory)**\*.zip" />
+  <!-- In build signing and publishing without a join point -->
+  <ItemGroup Condition="'$(SignAndPublishAsJoinPoint)' != 'true'">
+    <Artifact Include="$(ArtifactsPackagesDir)**\*.tar.gz;
+                       $(ArtifactsPackagesDir)**\*.zip;
+                       $(ArtifactsPackagesDir)**\*.deb;
+                       $(ArtifactsPackagesDir)**\*.rpm;
+                       $(ArtifactsPackagesDir)**\*.pkg;
+                       $(ArtifactsPackagesDir)**\*.exe;
+                       $(ArtifactsPackagesDir)**\*.msi"
+              Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz"
+              IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))">
+      <!-- Exclude wixpack.zip files from checksum generation -->
+      <ChecksumPath Condition="$([System.String]::Copy('%(Filename)%(Extension)').EndsWith('.wixpack.zip')) != 'true'">%(FullPath).sha512</ChecksumPath>
+    </Artifact>
+  </ItemGroup>
 
-    <ItemsToSignWithoutPaths Include="@(ItemsToSignWithPaths->'%(Filename)%(Extension)')" />
-    <ItemsToSignPostBuild Include="@(ItemsToSignWithoutPaths->Distinct())" Condition="'$(PostBuildSign)' == 'true'" />
-    <ItemsToSign Include="@(ItemsToSignWithPaths->Distinct())" Condition="'$(PostBuildSign)' != 'true'" />
+  <!-- Only the following artifacts should be signed.
+       Set SkipPublish=true as those artifacts are added again in Publishing.props. -->
+  <ItemGroup Condition="'$(SignAndPublishAsJoinPoint)' == 'true'">
+    <Artifact Include="$(DownloadDirectory)**\*.msi;
+                       $(DownloadDirectory)**\*.exe;
+                       $(DownloadDirectory)**\*.nupkg;
+                       $(DownloadDirectory)**\*.zip"
+              SkipPublish="true" />
   </ItemGroup>
+
 </Project>

eng/Subsets.props

@@ -198,7 +198,6 @@
     <SubsetName Include="Packs.Tests" Description="The framework pack tests." />
 
     <!-- Utility -->
-    <SubsetName Include="publish" OnDemand="true" Description="Generate asset manifests and prepare to publish to BAR." />
     <SubsetName Include="RegenerateDownloadTable" OnDemand="true" Description="Regenerates the nightly build download table" />
     <SubsetName Include="RegenerateThirdPartyNotices" OnDemand="true" Description="Regenerates the THIRD-PARTY-NOTICES.TXT file based on other repos' TPN files." />
     <SubsetName Include="tasks" OnDemand="true" Description="Build the repo local task projects." />
@@ -548,10 +547,6 @@
     <ProjectToBuild Include="@(TestProjectToBuild)" BuildInParallel="true" Test="true" Category="packs" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subset.Contains('+publish+'))">
-    <ProjectToBuild Include="$(InstallerProjectRoot)prepare-artifacts.proj" Pack="true" Category="publish" />
-  </ItemGroup>
-
   <!-- Utility -->
   <ItemGroup Condition="$(_subset.Contains('+regeneratedownloadtable+'))">
     <ProjectToBuild Include="$(RepositoryEngineeringDir)regenerate-download-table.proj" Pack="true" />

eng/pipelines/official/jobs/prepare-signed-artifacts.yml

@@ -46,10 +46,9 @@ jobs:
       checkDownloadedFiles: true
 
   - script: >-
-      build.cmd -ci
-      -subset publish
-      -configuration Release
+      build.cmd -sign -publish -ci -configuration Release
       /p:PublishRidAgnosticPackagesFromPlatform=${{ parameters.PublishRidAgnosticPackagesFromPlatform }}
+      /p:DownloadDirectory=$(Build.SourcesDirectory)\artifacts\PackageDownload\
       /p:OfficialBuildId=$(Build.BuildNumber)
       /p:SignType=$(SignType)
       /p:DotNetSignType=$(SignType)

eng/pipelines/runtime-official.yml

@@ -589,10 +589,6 @@ extends:
           - SourceBuild_linux_x64
           jobParameters:
             nameSuffix: PortableSourceBuild
-            postBuildSteps:
-              - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
-                parameters:
-                  name: SourceBuildPackages
             timeoutInMinutes: 95
 
       #