Bootstrap improvements

MSBuild.sln

@@ -84,6 +84,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Build.BuildCheck.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Build.Templates", "template_feed\Microsoft.Build.Templates.csproj", "{A86EE74A-AEF0-42ED-A5A7-7A54BC0773D8}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSBuild.Bootstrap.Utils", "src\MSBuild.Bootstrap.Utils\MSBuild.Bootstrap.Utils.csproj", "{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -916,6 +918,30 @@ Global
 		{A86EE74A-AEF0-42ED-A5A7-7A54BC0773D8}.Release|x64.Build.0 = Release|Any CPU
 		{A86EE74A-AEF0-42ED-A5A7-7A54BC0773D8}.Release|x86.ActiveCfg = Release|Any CPU
 		{A86EE74A-AEF0-42ED-A5A7-7A54BC0773D8}.Release|x86.Build.0 = Release|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Debug|ARM64.ActiveCfg = Debug|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Debug|ARM64.Build.0 = Debug|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Debug|x64.Build.0 = Debug|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Debug|x86.Build.0 = Debug|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.MachineIndependent|Any CPU.ActiveCfg = MachineIndependent|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.MachineIndependent|Any CPU.Build.0 = MachineIndependent|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.MachineIndependent|ARM64.ActiveCfg = MachineIndependent|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.MachineIndependent|ARM64.Build.0 = MachineIndependent|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.MachineIndependent|x64.ActiveCfg = MachineIndependent|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.MachineIndependent|x64.Build.0 = MachineIndependent|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.MachineIndependent|x86.ActiveCfg = MachineIndependent|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.MachineIndependent|x86.Build.0 = MachineIndependent|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Release|ARM64.ActiveCfg = Release|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Release|ARM64.Build.0 = Release|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Release|x64.ActiveCfg = Release|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Release|x64.Build.0 = Release|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Release|x86.ActiveCfg = Release|Any CPU
+		{E12F619A-80F1-4643-9AEC-D3D5C2FB8443}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

documentation/wiki/Bootstrap.md

@@ -0,0 +1,18 @@
+## Quick Intro
+The document describes the logic behind the bootstrap and testing capabilities for the fresh MSBuild bits.
+
+## History
+MSBuild supports two different environments: .NET and .NET Framework. To test changes for .NET, fresh bits were published (the actual target Publish run) to the MSBuild.Bootstrap folder. These bits, along with specific dependencies, were later copied to the bootstrap, making them ready for use with dotnet.exe. The executable is part of the .dotnet folder.
+
+## Current Implementation for .NET
+During the bootstrap phase, install-scripts are used to download the bits compatible with the current version. The logic for interacting with the scripts has been encapsulated in a separate MSBuild task: InstallDotNetCoreTask.cs. Here’s what happens under the hood:
+
+The SDK is downloaded to the bootstrap folder.
+Fresh MSBuild bits are then copied to this folder.
+The constructed SDK is used for both local end-to-end tests and CI runs.
+
+## Potential Cons
+The reliance on downloading the SDK from a remote source requires an internet connection. For the initial build of the repository, this doesn't change as the SDK is always downloaded to the .dotnet folder first. However, for subsequent runs, the SDK will need to be downloaded again, which could be problematic in environments with limited or no internet connectivity.
+
+## Pros
+This approach simplifies testing MSBuild as part of dotnet by providing a ready and reliable environment without needing to patch anything into a globally installed SDK, as was previously required.

eng/BootStrapMsBuild.props

@@ -1,20 +1,23 @@
 <Project>
 
-  <!--
-  Construct a location of MSBuild bootstrap folder - to be used for deployment and for tests
-   relying on bootstrapped MSBuild
-   -->
+  <!-- Construct a location of MSBuild bootstrap folder - to be used for deployment and for tests relying on bootstrapped MSBuild -->
+
+  <PropertyGroup Condition="!$(TargetFramework.StartsWith('net4'))">
+    <NetVersion>8.0.302</NetVersion>
+  </PropertyGroup>
 
   <PropertyGroup>
     <BootstrapDestination>$(ArtifactsBinDir)bootstrap\</BootstrapDestination>
     <BootstrapDestination Condition="'$(Platform)' == 'x64' or '$(Platform)' == 'arm64'">$(BootstrapDestination)$(Platform)\</BootstrapDestination>
+
+    <!-- This path is used for Windows Full test run and it points to net472 during execution-->
     <BootstrapDestination>$(BootstrapDestination)$(TargetFramework.ToLowerInvariant())\MSBuild\</BootstrapDestination>
   </PropertyGroup>
-  
+
   <PropertyGroup Condition="$(TargetFramework.StartsWith('net4'))">
     <BootstrapBinaryDestination>$(BootstrapDestination)$(TargetMSBuildToolsVersion)\Bin</BootstrapBinaryDestination>
   </PropertyGroup>
-  
+
   <PropertyGroup Condition="!$(TargetFramework.StartsWith('net4'))">
     <BootstrapBinaryDestination>$(BootstrapDestination)</BootstrapBinaryDestination>
   </PropertyGroup>

eng/BootStrapMsBuild.targets

@@ -202,56 +202,32 @@
            AlwaysCreate="true" />
   </Target>
 
+  <!-- The task allow to download sdk bits for the specified version. It will be used later to bootstrap the runnable MSBuild. -->
+  <UsingTask TaskName="MSBuild.Bootstrap.Utils.Tasks.InstallDotNetCoreTask" AssemblyFile="$(ArtifactsBinDir)MSBuild.Bootstrap.Utils\$(Configuration)\netstandard2.0\MSBuild.Bootstrap.Utils.dll" TaskFactory="TaskHostFactory" />
   <Target Name="BootstrapNetCore" DependsOnTargets="CleanBootstrapFolder">
 
-    <!-- Publish the project first.  The more obvious way to do this would be to depend on the Publish target,
-         but that causes a target dependency cycle, even though it seems like AfterTargets="AfterBuild" should
-         probably not count as a link in the cycle. -->
-    <MSBuild Projects="$(MSBuildProjectFile)" Targets="Publish" BuildInParallel="$(BuildInParallel)" />
+    <PropertyGroup>
+      <InstallDir>$(ArtifactsBinDir)bootstrap\$(TargetFramework)\</InstallDir>
+    </PropertyGroup>
 
-    <ItemGroup>
-      <!-- Copy all items from the publish folder to the bootstrap folder.  We might be able to just use the published
-            version as the bootstrapped version, but the extra separation here seems like it could be valuable. -->
-      <DeployedItems Include="$(PublishDir)\**\*.*" />
-
-      <NuGetSdkResolverManifest Include= "$(RepoRoot)src\MSBuild\SdkResolvers\Standalone\Microsoft.Build.NuGetSdkResolver.xml" />
-      <InstalledSdks Include="$(DOTNET_INSTALL_DIR)\sdk\$(DotNetCliVersion)\Sdks\**\*.*" />
-      <InstalledExtensions Include="$(DOTNET_INSTALL_DIR)\sdk\$(DotNetCliVersion)\Current\**\*.*" Exclude="$(DOTNET_INSTALL_DIR)\sdk\$(DotNetCliVersion)\Current\Microsoft.Common.props" />
-    </ItemGroup>
-    <Copy SourceFiles="@(DeployedItems)"
-          DestinationFolder="$(BootstrapDestination)%(RecursiveDir)" />
-
-    <Copy SourceFiles="@(NuGetSdkResolverManifest)"
-          DestinationFolder="$(BootstrapDestination)SdkResolvers\Microsoft.Build.NuGetSdkResolver" />
-
-    <Copy SourceFiles="$(RuntimeIdentifierGraphPath)"
-          DestinationFolder="$(BootstrapDestination)" />
-
-    <Copy SourceFiles="@(InstalledSdks)"
-          DestinationFiles="@(InstalledSdks -> '$(BootstrapDestination)Sdks\%(RecursiveDir)%(Filename)%(Extension)')" />
-
-    <!-- The .NET SDK has a dependency on DependencyModel, but relies on having it in the final
-         MSBuild.deps.json, which differs from ours because it's generated in the SDK repo.
+    <InstallDotNetCoreTask DotNetInstallScriptRootPath="$(DotNetRoot)" InstallDir="$(InstallDir)" Version="$(NetVersion)" DotNetInstallBaseUrl="https://raw.githubusercontent.com/dotnet/install-scripts/main/src/"/>
+  </Target>
 
-         Copy it from "next to MSBuild" in the pre-bootstrap SDK to our little weirdo bootstrap
-         layout next to the SDK tasks, so it can get loaded by the SDK tasks that need it. -->
-    <Copy SourceFiles="$(DOTNET_INSTALL_DIR)\sdk\$(DotNetCliVersion)\Microsoft.Extensions.DependencyModel.dll"
-          DestinationFolder="$(BootstrapDestination)Sdks\Microsoft.NET.Sdk\tools\$(LatestDotNetCoreForMSBuild)" />
+  <Target Name="BootstrapNetCorePatch" AfterTargets="BootstrapNetCore">
 
-    <Copy SourceFiles="@(InstalledExtensions)"
-          DestinationFolder="$(BootstrapDestination)Current\%(RecursiveDir)" />
+    <ItemGroup>
+     <!-- *.deps.json are excluded because they will cause the conflicts on an attempt to build solution with the bootstraped bits. -->
+      <FreshlyBuiltNetBinaries Include="$(OutDir)**\*.*" Exclude="$(OutDir)**\*.deps.json" />
+    </ItemGroup>
 
+    <!-- The copying of these dependencies is required by bootstrap\**\sdk\**\NuGet.RestoreEx.targets. Otherwise NuGet.Build.Tasks.dll can not be found. -->
     <Copy SourceFiles="@(_NuGetRuntimeDependencies)"
-          DestinationFolder="$(BootstrapDestination)" />
-
-    <Copy SourceFiles="$(RepoRoot)src\MSBuild.Bootstrap\RedirectNuGetConsoleProcess.After.Microsoft.Common.targets"
-          DestinationFolder="$(BootstrapDestination)\Current\Microsoft.Common.targets\ImportAfter" />
+          DestinationFolder="$(InstallDir)sdk\$(NetVersion)\"
+          SkipUnchangedFiles="true" />
 
-    <Copy SourceFiles="$(RepoRoot)src\MSBuild.Bootstrap\RedirectNuGetConsoleProcess.After.Microsoft.Common.targets"
-          DestinationFolder="$(BootstrapDestination)\Current\SolutionFile\ImportAfter" />
+    <Copy SourceFiles="@(FreshlyBuiltNetBinaries)"
+          DestinationFiles="@(FreshlyBuiltNetBinaries->'$(InstallDir)sdk\$(NetVersion)\%(RecursiveDir)%(Filename)%(Extension)')" />
 
-    <!-- Disable workload resolver until we can figure out whether it can work in the bootstrap
-         https://github.com/dotnet/msbuild/issues/6566 -->
-    <Touch Files="$(BootstrapDestination)\DisableWorkloadResolver.sentinel" AlwaysCreate="true" />
   </Target>
+
 </Project>

eng/cibuild_bootstrapped_msbuild.ps1

@@ -84,8 +84,11 @@ try {
   }
   else
   {
-    $buildToolPath = $dotnetExePath
-    $buildToolCommand = Join-Path $bootstrapRoot "net8.0\MSBuild\MSBuild.dll"
+    $buildToolPath = Join-Path $bootstrapRoot "net8.0\dotnet.exe"
+
+    # Must be consistent with the version in BootStrapMsBuild.props
+    $buildToolCommand = Join-Path $bootstrapRoot "net8.0\sdk\8.0.302\MSBuild.dll";
+
     $buildToolFramework = "net8.0"
   }
 

eng/cibuild_bootstrapped_msbuild.sh

@@ -58,8 +58,10 @@ bootstrapRoot="$Stage1Dir/bin/bootstrap"
 
 if [ $host_type = "core" ]
 then
-  _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
-  _InitializeBuildToolCommand="$bootstrapRoot/net8.0/MSBuild/MSBuild.dll"
+  _InitializeBuildTool="$bootstrapRoot/net8.0/dotnet"
+
+  # Must be consistent with the version in BootStrapMsBuild.props
+  _InitializeBuildToolCommand="$bootstrapRoot/net8.0/sdk/8.0.302/MSBuild.dll"
   _InitializeBuildToolFramework="net8.0"
 else
   echo "Unsupported hostType ($host_type)"

src/MSBuild.Bootstrap.Utils/MSBuild.Bootstrap.Utils.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Build.Utilities.Core" VersionOverride="15.5.180" ExcludeAssets="runtime" PrivateAssets="all" />
+  </ItemGroup>
+
+</Project>