Bootstrap improvements

documentation/release-checklist.md

@@ -56,6 +56,7 @@ https://ceapex.visualstudio.com/CEINTL/_workitems/edit/957875 (DONE)
 - [ ] Get M2 or QB approval as necessary per the VS schedule
 - [ ]  Merge to VS (babysit the automatically generated VS insertion PR https://devdiv.visualstudio.com/DevDiv/_git/VS/pullrequests for the MSBuild commit noted in above step): {{URL_OF_VS_INSERTION}}
 - [ ] Update the PackageValidationBaselineVersion to the latest released version ({{THIS_RELEASE_VERSION}}.0) - this might require temporary addition of the [build artifacts feed](https://github.com/dotnet/msbuild/blob/29397b577e3ec0fe0c7650c3ab0400909655dc88/NuGet.config#L9) as the new version is not yet added to the official feeds (this is post release). This can trigger a high severity CG error (https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/how-to-securely-configure-package-source-files) - however it should be fine to keep this temporary feed untill the release.
+- [ ] Update the requested SDK version for bootstrap folder (the `NetVersion` property in [BootStrapMsBuild.props](https://github.com/dotnet/msbuild/blob/main/eng/BootStrapMsBuild.props)) and buildToolCommand/_InitializeBuildToolCommand values in cibuild_bootstrapped_msbuild scripts if a fresh sdk was released (released runtimes and associated sdk versions can be checked here - https://dotnet.microsoft.com/en-us/download/visual-studio-sdks - make sure to allways check the details of the appropriate targetted version of .NET for the matchin latest version of SDK).
 
 ## ASAP On/After GA:
 

documentation/wiki/Bootstrap.md

@@ -0,0 +1,22 @@
+# Bootstrap MSBuild
+
+Because the binaries and build logic in this repo aren't sufficient to build real-world projects, we need a test environment that mimics the real-world combinations of MSBuild, Roslyn compilers, and other things that combine in the .NET SDK and in Visual Studio to produce a functional build environment. We call this the "bootstrap".
+
+## Quick Intro
+The document describes the logic behind the bootstrap and testing capabilities for the fresh MSBuild bits.
+
+## History
+MSBuild supports two different environments: .NET and .NET Framework. To test changes for .NET, fresh bits were published (the actual target Publish run) to the MSBuild.Bootstrap folder. These bits, along with specific dependencies, were later combined with parts of the .NET SDK that was used to build MSBuild to the bootstrap, making them ready for use with dotnet.exe. To execute the bootstrap MSBuild, you'd combine the `dotnet.exe` muxer from the .dotnet folder with the path to the bootstrap's `MSBuild.dll`.
+
+## Current Implementation for .NET
+During the bootstrap phase, install-scripts are used to download a full copy of the .NET SDK compatible with the current version. The logic for interacting with the scripts has been encapsulated in a separate MSBuild task: InstallDotNetCoreTask.cs. Here’s what happens under the hood:
+
+The SDK is downloaded to the bootstrap folder.
+Fresh MSBuild bits are then copied to this folder.
+The constructed SDK is used for both local end-to-end tests and CI runs.
+
+## Potential Cons
+The reliance on downloading the SDK from a remote source requires an internet connection. For the initial build of the repository, this doesn't change as the SDK is always downloaded to the .dotnet folder first. For subsequent runs, the SDK in bootstrap will be downloaded again only **if the requested version was changed**.
+
+## Pros
+This approach simplifies testing MSBuild as part of dotnet by providing a ready and reliable environment without needing to patch anything into a globally installed SDK, as was previously required.

eng/BootStrapMsBuild.props

@@ -1,20 +1,23 @@
 <Project>
 
-  <!--
-  Construct a location of MSBuild bootstrap folder - to be used for deployment and for tests
-   relying on bootstrapped MSBuild
-   -->
+  <!-- Construct a location of MSBuild bootstrap folder - to be used for deployment and for tests relying on bootstrapped MSBuild -->
+
+  <PropertyGroup Condition="!$(TargetFramework.StartsWith('net4'))">
+    <NetVersion>8.0.302</NetVersion>
+  </PropertyGroup>
 
   <PropertyGroup>
     <BootstrapDestination>$(ArtifactsBinDir)bootstrap\</BootstrapDestination>
     <BootstrapDestination Condition="'$(Platform)' == 'x64' or '$(Platform)' == 'arm64'">$(BootstrapDestination)$(Platform)\</BootstrapDestination>
+
+    <!-- This path is used for Windows Full test run and it points to net472 during execution-->
     <BootstrapDestination>$(BootstrapDestination)$(TargetFramework.ToLowerInvariant())\MSBuild\</BootstrapDestination>
   </PropertyGroup>
-  
+
   <PropertyGroup Condition="$(TargetFramework.StartsWith('net4'))">
     <BootstrapBinaryDestination>$(BootstrapDestination)$(TargetMSBuildToolsVersion)\Bin</BootstrapBinaryDestination>
   </PropertyGroup>
-  
+
   <PropertyGroup Condition="!$(TargetFramework.StartsWith('net4'))">
     <BootstrapBinaryDestination>$(BootstrapDestination)</BootstrapBinaryDestination>
   </PropertyGroup>

eng/BootStrapMsBuild.targets

@@ -202,56 +202,40 @@
            AlwaysCreate="true" />
   </Target>
 
-  <Target Name="BootstrapNetCore" DependsOnTargets="CleanBootstrapFolder">
+  <!-- The task allow to download sdk bits for the specified version. It will be used later to bootstrap the runnable MSBuild. -->
+  <UsingTask TaskName="InstallDotNetCoreTask"
+             AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll"
+             TaskFactory="RoslynCodeTaskFactory">
+    <Task>
+      <Code Source="$(MSBuildThisFileDirectory)..\src\MSBuild.Bootstrap.Utils\Tasks\InstallDotNetCoreTask.cs" Language="cs" />
+    </Task>
+  </UsingTask>
+  <Target Name="AcquireSdk"
+          BeforeTargets="PrepareForBuild"
+          Condition="!$(TargetFramework.StartsWith('net4'))">
+
+    <PropertyGroup>
+      <InstallDir>$(ArtifactsBinDir)bootstrap\core\</InstallDir>
+    </PropertyGroup>
+
+    <InstallDotNetCoreTask DotNetInstallScriptRootPath="$(DotNetRoot)" InstallDir="$(InstallDir)" Version="$(NetVersion)"/>
+  </Target>
 
-    <!-- Publish the project first.  The more obvious way to do this would be to depend on the Publish target,
-         but that causes a target dependency cycle, even though it seems like AfterTargets="AfterBuild" should
-         probably not count as a link in the cycle. -->
-    <MSBuild Projects="$(MSBuildProjectFile)" Targets="Publish" BuildInParallel="$(BuildInParallel)" />
+  <Target Name="BootstrapNetCore" DependsOnTargets="AcquireSdk">
 
     <ItemGroup>
-      <!-- Copy all items from the publish folder to the bootstrap folder.  We might be able to just use the published
-            version as the bootstrapped version, but the extra separation here seems like it could be valuable. -->
-      <DeployedItems Include="$(PublishDir)\**\*.*" />
-
-      <NuGetSdkResolverManifest Include= "$(RepoRoot)src\MSBuild\SdkResolvers\Standalone\Microsoft.Build.NuGetSdkResolver.xml" />
-      <InstalledSdks Include="$(DOTNET_INSTALL_DIR)\sdk\$(DotNetCliVersion)\Sdks\**\*.*" />
-      <InstalledExtensions Include="$(DOTNET_INSTALL_DIR)\sdk\$(DotNetCliVersion)\Current\**\*.*" Exclude="$(DOTNET_INSTALL_DIR)\sdk\$(DotNetCliVersion)\Current\Microsoft.Common.props" />
+     <!-- *.deps.json are excluded because they will cause the conflicts on an attempt to build solution with the bootstraped bits. -->
+      <FreshlyBuiltNetBinaries Include="$(OutDir)**\*.*" Exclude="$(OutDir)**\*.deps.json" />
     </ItemGroup>
-    <Copy SourceFiles="@(DeployedItems)"
-          DestinationFolder="$(BootstrapDestination)%(RecursiveDir)" />
-
-    <Copy SourceFiles="@(NuGetSdkResolverManifest)"
-          DestinationFolder="$(BootstrapDestination)SdkResolvers\Microsoft.Build.NuGetSdkResolver" />
-
-    <Copy SourceFiles="$(RuntimeIdentifierGraphPath)"
-          DestinationFolder="$(BootstrapDestination)" />
-
-    <Copy SourceFiles="@(InstalledSdks)"
-          DestinationFiles="@(InstalledSdks -> '$(BootstrapDestination)Sdks\%(RecursiveDir)%(Filename)%(Extension)')" />
-
-    <!-- The .NET SDK has a dependency on DependencyModel, but relies on having it in the final
-         MSBuild.deps.json, which differs from ours because it's generated in the SDK repo.
-
-         Copy it from "next to MSBuild" in the pre-bootstrap SDK to our little weirdo bootstrap
-         layout next to the SDK tasks, so it can get loaded by the SDK tasks that need it. -->
-    <Copy SourceFiles="$(DOTNET_INSTALL_DIR)\sdk\$(DotNetCliVersion)\Microsoft.Extensions.DependencyModel.dll"
-          DestinationFolder="$(BootstrapDestination)Sdks\Microsoft.NET.Sdk\tools\$(LatestDotNetCoreForMSBuild)" />
-
-    <Copy SourceFiles="@(InstalledExtensions)"
-          DestinationFolder="$(BootstrapDestination)Current\%(RecursiveDir)" />
 
+    <!-- The copying of these dependencies is required by bootstrap\**\sdk\**\NuGet.RestoreEx.targets. Otherwise NuGet.Build.Tasks.dll can not be found. -->
     <Copy SourceFiles="@(_NuGetRuntimeDependencies)"
-          DestinationFolder="$(BootstrapDestination)" />
-
-    <Copy SourceFiles="$(RepoRoot)src\MSBuild.Bootstrap\RedirectNuGetConsoleProcess.After.Microsoft.Common.targets"
-          DestinationFolder="$(BootstrapDestination)\Current\Microsoft.Common.targets\ImportAfter" />
+          DestinationFolder="$(InstallDir)sdk\$(NetVersion)\"
+          SkipUnchangedFiles="true" />
 
-    <Copy SourceFiles="$(RepoRoot)src\MSBuild.Bootstrap\RedirectNuGetConsoleProcess.After.Microsoft.Common.targets"
-          DestinationFolder="$(BootstrapDestination)\Current\SolutionFile\ImportAfter" />
+    <Copy SourceFiles="@(FreshlyBuiltNetBinaries)"
+          DestinationFiles="@(FreshlyBuiltNetBinaries->'$(InstallDir)sdk\$(NetVersion)\%(RecursiveDir)%(Filename)%(Extension)')" />
 
-    <!-- Disable workload resolver until we can figure out whether it can work in the bootstrap
-         https://github.com/dotnet/msbuild/issues/6566 -->
-    <Touch Files="$(BootstrapDestination)\DisableWorkloadResolver.sentinel" AlwaysCreate="true" />
   </Target>
+
 </Project>

eng/cibuild_bootstrapped_msbuild.ps1

@@ -84,8 +84,8 @@ try {
   }
   else
   {
-    $buildToolPath = $dotnetExePath
-    $buildToolCommand = Join-Path $bootstrapRoot "net8.0\MSBuild\MSBuild.dll"
+    $buildToolPath = Join-Path $bootstrapRoot "core\dotnet.exe"
+    $buildToolCommand = Join-Path $bootstrapRoot "core\sdk\8.0.302\MSBuild.dll"
     $buildToolFramework = "net8.0"
   }
 

eng/cibuild_bootstrapped_msbuild.sh

@@ -58,8 +58,8 @@ bootstrapRoot="$Stage1Dir/bin/bootstrap"
 
 if [ $host_type = "core" ]
 then
-  _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
-  _InitializeBuildToolCommand="$bootstrapRoot/net8.0/MSBuild/MSBuild.dll"
+  _InitializeBuildTool="$bootstrapRoot/core/dotnet"
+  _InitializeBuildToolCommand="$bootstrapRoot/core/sdk/8.0.302/MSBuild.dll"
   _InitializeBuildToolFramework="net8.0"
 else
   echo "Unsupported hostType ($host_type)"

src/MSBuild.Bootstrap.Utils/MSBuild.Bootstrap.Utils.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Build.Utilities.Core" VersionOverride="15.5.180" ExcludeAssets="runtime" PrivateAssets="all" />
+  </ItemGroup>
+
+</Project>

src/MSBuild.Bootstrap.Utils/Tasks/InstallDotNetCoreTask.cs

@@ -0,0 +1,172 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.IO;
+using System.Net.Http;
+using System.Runtime.InteropServices;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+using AsyncTasks = System.Threading.Tasks;
+
+namespace MSBuild.Bootstrap.Utils.Tasks
+{
+    /// <summary>
+    /// This task is designed to automate the installation of .NET Core SDK.
+    /// It downloads the appropriate installation script and executes it to install the specified version of .NET Core SDK.
+    /// </summary>
+    public sealed class InstallDotNetCoreTask : ToolTask
+    {
+        private const string ScriptName = "dotnet-install";
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="InstallDotNetCoreTask"/> class.
+        /// </summary>
+        public InstallDotNetCoreTask()
+        {
+            InstallDir = string.Empty;
+            DotNetInstallScriptRootPath = string.Empty;
+            Version = string.Empty;
+        }
+
+        /// <summary>
+        /// Gets or sets the directory where the .NET Core SDK should be installed. This property is required.
+        /// </summary>
+        [Required]
+        public string InstallDir { get; set; }
+
+        /// <summary>
+        /// Gets or sets the root path where the .NET Core installation script is located. This property is required.
+        /// </summary>
+        [Required]
+        public string DotNetInstallScriptRootPath { get; set; }
+
+        /// <summary>
+        /// Gets or sets the version of the .NET Core SDK to be installed. This property is required.
+        /// </summary>
+        [Required]
+        public string Version { get; set; }
+
+        /// <summary>
+        /// Gets or sets the base URL for downloading the .NET Core installation script. The default value is "https://dot.net/v1/".
+        /// </summary>
+        public string DotNetInstallBaseUrl { get; set; } = "https://dot.net/v1/";
+
+        private bool IsWindows => RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
+
+        protected override string ToolName => IsWindows ? "powershell.exe" : "/bin/bash";
+
+        /// <summary>
+        /// Executes the task, downloading and running the .NET Core installation script.
+        /// </summary>
+        /// <returns>True if the task succeeded; otherwise, false.</returns>
+        public override bool Execute()
+        {
+            if (Directory.Exists(Path.Combine(InstallDir, "sdk", Version)))
+            {
+                // no need to download sdk again, it exists locally
+                return true;
+            }
+
+            ScriptExecutionSettings executionSettings = SetupScriptsExecutionSettings();
+            if (!File.Exists(executionSettings.ScriptsFullPath))
+            {
+                AsyncTasks.Task.Run(() => DownloadScriptAsync(executionSettings.ScriptName, executionSettings.ScriptsFullPath)).GetAwaiter().GetResult();
+            }
+
+            MakeScriptExecutable(executionSettings.ScriptsFullPath);
+
+            return RunScript(executionSettings);
+        }
+
+        protected override string GenerateFullPathToTool() => ToolName;
+
+        /// <summary>
+        /// Downloads the .NET Core installation script asynchronously from the specified URL.
+        /// </summary>
+        /// <param name="scriptName">The name of the script to download.</param>
+        /// <param name="scriptPath">The path where the script will be saved.</param>
+        private async AsyncTasks.Task DownloadScriptAsync(string scriptName, string scriptPath)
+        {
+            using (HttpClient client = new HttpClient())
+            {
+                HttpResponseMessage response = await client.GetAsync($"{DotNetInstallBaseUrl}{scriptName}").ConfigureAwait(false);
+                if (response.IsSuccessStatusCode)
+                {
+                    string scriptContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
+                    if (!string.IsNullOrEmpty(scriptContent))
+                    {
+                        File.WriteAllText(scriptPath, scriptContent);
+                    }
+                }
+                else
+                {
+                    Log.LogError($"Install-scripts download from {DotNetInstallBaseUrl} error. Status code: {response.StatusCode}.");
+                }
+            }
+        }
+
+        /// <summary>
+        /// Makes the installation script executable on non-Windows platforms.
+        /// </summary>
+        /// <param name="scriptPath">The path of the script to make executable.</param>
+        private void MakeScriptExecutable(string scriptPath)
+        {
+            if (!IsWindows)
+            {
+                int exitCode = ExecuteTool("/bin/chmod", string.Empty, $"+x {scriptPath}");
+                if (exitCode != 0)
+                {
+                    Log.LogError($"Install-scripts can not be made executable due to the errors reported above.");
+                }
+            }
+        }
+
+        /// <summary>
+        /// Runs the .NET Core installation script with the specified settings.
+        /// </summary>
+        /// <param name="executionSettings">The settings required for script execution.</param>
+        /// <returns>True if the script executed successfully; otherwise, false.</returns>
+        private bool RunScript(ScriptExecutionSettings executionSettings)
+        {
+            if (!Log.HasLoggedErrors)
+            {
+                int exitCode = ExecuteTool(ToolName, string.Empty, executionSettings.ExecutableArgs);
+
+                if (exitCode != 0)
+                {
+                    Log.LogError($"Install-scripts was not executed successfully.");
+                }
+            }
+
+            return !Log.HasLoggedErrors;
+        }
+
+        /// <summary>
+        /// Sets up the settings required for executing the .NET Core installation script.
+        /// </summary>
+        /// <returns>The settings required for script execution.</returns>
+        private ScriptExecutionSettings SetupScriptsExecutionSettings()
+        {
+            string scriptExtension = IsWindows ? "ps1" : "sh";
+            string scriptPath = Path.Combine(DotNetInstallScriptRootPath, $"{ScriptName}.{scriptExtension}");
+            string scriptArgs = IsWindows
+                ? $"-NoProfile -ExecutionPolicy Bypass -File {scriptPath} -Version {Version} -InstallDir {InstallDir}"
+                : $"{scriptPath} --version {Version} --install-dir {InstallDir}";
+
+            return new ScriptExecutionSettings($"{ScriptName}.{scriptExtension}", scriptPath, scriptArgs);
+        }
+
+        /// <summary>
+        /// A private struct to hold settings for script execution.
+        /// </summary>
+        private readonly struct ScriptExecutionSettings(string scriptName, string scriptsFullPath, string executableArgs)
+        {
+            public string ScriptName { get; } = scriptName;
+
+            public string ScriptsFullPath { get; } = scriptsFullPath;
+
+            public string ExecutableArgs { get; } = executableArgs;
+        }
+    }
+}

src/MSBuild.Bootstrap/MSBuild.Bootstrap.csproj

@@ -18,6 +18,7 @@
     <ProjectReference Include="..\Framework\Microsoft.Build.Framework.csproj" />
     <ProjectReference Include="..\Tasks\Microsoft.Build.Tasks.csproj" />
     <ProjectReference Include="..\Utilities\Microsoft.Build.Utilities.csproj" />
+    <ProjectReference Include="..\MSBuild.Bootstrap.Utils\MSBuild.Bootstrap.Utils.csproj" ReferenceOutputAssembly="false" />
   </ItemGroup>
   <ItemGroup>
     <!-- This file is needed so the dotnet CLI knows how to map preview SDK versions to tfms (because tfms do not have preview information on them) -->
@@ -47,7 +48,6 @@
 
   <Import Project="..\Package\GetBinPaths.targets" Condition="$(TargetFramework.StartsWith('net4'))" />
   <Import Project="..\Package\GetBinPaths.Arm64.targets" Condition="$(TargetFramework.StartsWith('net4'))" />
-
   <Import Project="$(RepoRoot)eng\BootStrapMsBuild.targets" />
 
-</Project>
+</Project>