Update Roslyn and use new step tracking API in incremental tests.

eng/Versions.props

@@ -22,20 +22,27 @@
     <LastReleasedStableAssemblyVersion>$(AssemblyVersion)</LastReleasedStableAssemblyVersion>
     <UsingToolMicrosoftNetCompilers>true</UsingToolMicrosoftNetCompilers>
   </PropertyGroup>
+  <!--
+    For source generator support we need to target multiple versions of Rolsyn in order to be able to run on older versions of Roslyn.
+    We pin these versions as we need to match them exactly for any scenarios that run Roslyn on .NET Framework, like Visual Studio.
+  -->
   <PropertyGroup>
-    <!-- For source generator support we need to target multiple versions of Rolsyn in order to be able to run on older versions of Roslyn -->
-    <MicrosoftCodeAnalysisCSharpWorkspacesVersion_3_11>3.11.0</MicrosoftCodeAnalysisCSharpWorkspacesVersion_3_11>
-    <MicrosoftCodeAnalysisCSharpWorkspacesVersion>4.0.1</MicrosoftCodeAnalysisCSharpWorkspacesVersion>
-    <MicrosoftCodeAnalysisVersion>4.0.1</MicrosoftCodeAnalysisVersion>
+    <MicrosoftCodeAnalysisVersion_3_11>3.11.0</MicrosoftCodeAnalysisVersion_3_11>
+    <MicrosoftCodeAnalysisVersion_4_0>4.0.1</MicrosoftCodeAnalysisVersion_4_0>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Code analysis dependencies -->
-    <MicrosoftCodeAnalysisAnalyzersVersion>3.3.2</MicrosoftCodeAnalysisAnalyzersVersion>
-    <MicrosoftCodeAnalysisCSharpCodeStyleVersion>4.0.1</MicrosoftCodeAnalysisCSharpCodeStyleVersion>
-    <MicrosoftCodeAnalysisCSharpVersion>4.0.1</MicrosoftCodeAnalysisCSharpVersion>
+    <MicrosoftCodeAnalysisAnalyzersVersion>3.3.3</MicrosoftCodeAnalysisAnalyzersVersion>
+    <MicrosoftCodeAnalysisCSharpCodeStyleVersion>4.2.0-2.22128.1</MicrosoftCodeAnalysisCSharpCodeStyleVersion>
+    <MicrosoftCodeAnalysisCSharpWorkspacesVersion>4.2.0-2.22128.1</MicrosoftCodeAnalysisCSharpWorkspacesVersion>
+    <MicrosoftCodeAnalysisCSharpVersion>4.2.0-2.22128.1</MicrosoftCodeAnalysisCSharpVersion>
     <MicrosoftCodeAnalysisNetAnalyzersVersion>7.0.0-preview1.22119.1</MicrosoftCodeAnalysisNetAnalyzersVersion>
-    <!-- TODO: Remove pinned compiler version once arcade supplies runtime with a compiler capable of handling !! -->
-    <MicrosoftNetCompilersToolsetVersion>4.2.0-2.22105.4</MicrosoftNetCompilersToolsetVersion>
+    <MicrosoftCodeAnalysisVersion>4.2.0-2.22128.1</MicrosoftCodeAnalysisVersion>
+    <!--
+      TODO: Remove pinned compiler version once arcade supplies runtime with a compiler capable of handling !!
+      and has https://github.com/dotnet/roslyn/pull/59776
+    -->
+    <MicrosoftNetCompilersToolsetVersion>4.2.0-2.22128.1</MicrosoftNetCompilersToolsetVersion>
     <!-- SDK dependencies -->
     <MicrosoftDotNetCompatibilityVersion>2.0.0-alpha.1.21525.11</MicrosoftDotNetCompatibilityVersion>
     <!-- Arcade dependencies -->

eng/formatting/format.sh

@@ -14,6 +14,7 @@ if [ -n "$NATIVE_FILES" ]; then
     # Add back the modified files to staging
     echo "$NATIVE_FILES" | xargs git add
 fi
+
 if [ -n "$MANAGED_FILES" ]; then
     # Format all selected files
     echo "$MANAGED_FILES" | cat | xargs | sed -e 's/ /,/g' | dotnet format --no-restore --include -

src/libraries/Common/src/Interop/OSX/Interop.Libraries.cs

@@ -15,6 +15,5 @@ internal static partial class Libraries
         internal const string SystemConfigurationLibrary = "/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration";
         internal const string AppleCryptoNative = "libSystem.Security.Cryptography.Native.Apple";
         internal const string MsQuic = "libmsquic.dylib";
-        internal const string libc = "libc";
     }
 }

src/libraries/Common/src/Interop/Unix/Interop.Libraries.cs

@@ -5,7 +5,7 @@ internal static partial class Interop
 {
     internal static partial class Libraries
     {
-        internal const string Libc = "libc";
+        internal const string libc = "libc";
 
         // Shims
         internal const string SystemNative = "libSystem.Native";

src/libraries/Common/src/Interop/Unix/libc/Interop.GetParentPid.cs

@@ -7,7 +7,7 @@ internal static partial class Interop
 {
     internal static partial class @libc
     {
-        [GeneratedDllImport(Libraries.Libc, EntryPoint = "getppid")]
+        [GeneratedDllImport(Libraries.libc, EntryPoint = "getppid")]
         internal static partial int GetParentPid();
     }
 }

src/libraries/Common/tests/TestUtilities/System/PlatformDetection.Unix.cs

@@ -51,7 +51,7 @@ public static partial class PlatformDetection
         public static bool IsNotFedoraOrRedHatFamily => !IsFedora && !IsRedHatFamily;
         public static bool IsNotDebian10 => !IsDebian10;
 
-        public static bool IsSuperUser => IsBrowser || IsWindows ? false : Libc.geteuid() == 0;
+        public static bool IsSuperUser => IsBrowser || IsWindows ? false : libc.geteuid() == 0;
 
         public static Version OpenSslVersion => !IsOSXLike && !IsWindows && !IsAndroid ?
             GetOpenSslVersion() :
@@ -77,7 +77,7 @@ public static string LibcRelease
 
                 try
                 {
-                    return Marshal.PtrToStringAnsi(Libc.gnu_get_libc_release());
+                    return Marshal.PtrToStringAnsi(libc.gnu_get_libc_release());
                 }
                 catch (Exception e) when (e is DllNotFoundException || e is EntryPointNotFoundException)
                 {
@@ -101,7 +101,7 @@ public static string LibcVersion
 
                 try
                 {
-                    return Marshal.PtrToStringAnsi(Libc.gnu_get_libc_version());
+                    return Marshal.PtrToStringAnsi(libc.gnu_get_libc_version());
                 }
                 catch (Exception e) when (e is DllNotFoundException || e is EntryPointNotFoundException)
                 {
@@ -324,7 +324,7 @@ private struct DistroInfo
             public Version VersionId { get; set; }
         }
 
-        private static partial class Libc
+        private static partial class @libc
         {
             [GeneratedDllImport("libc", SetLastError = true)]
             public static unsafe partial uint geteuid();

src/libraries/Microsoft.Extensions.Logging.Abstractions/gen/Microsoft.Extensions.Logging.Generators.Roslyn3.11.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <AnalyzerRoslynVersion>3.11</AnalyzerRoslynVersion>
-    <RoslynApiVersion>$(MicrosoftCodeAnalysisCSharpWorkspacesVersion_3_11)</RoslynApiVersion>
+    <RoslynApiVersion>$(MicrosoftCodeAnalysisVersion_3_11)</RoslynApiVersion>
   </PropertyGroup>
 
   <Import Project="Microsoft.Extensions.Logging.Generators.targets" />

src/libraries/Microsoft.Extensions.Logging.Abstractions/gen/Microsoft.Extensions.Logging.Generators.Roslyn4.0.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <AnalyzerRoslynVersion>4.0</AnalyzerRoslynVersion>
-    <RoslynApiVersion>$(MicrosoftCodeAnalysisCSharpWorkspacesVersion)</RoslynApiVersion>
+    <RoslynApiVersion>$(MicrosoftCodeAnalysisVersion_4_0)</RoslynApiVersion>
     <DefineConstants>$(DefineConstants);ROSLYN4_0_OR_GREATER</DefineConstants>
   </PropertyGroup>
 

src/libraries/Microsoft.Extensions.Logging.Abstractions/tests/Microsoft.Extensions.Logging.Generators.Tests/Microsoft.Extensions.Logging.Generators.Roslyn3.11.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <RoslynApiVersion>$(MicrosoftCodeAnalysisCSharpWorkspacesVersion_3_11)</RoslynApiVersion>
+    <RoslynApiVersion>$(MicrosoftCodeAnalysisVersion_3_11)</RoslynApiVersion>
     <IsHighAotMemoryUsageTest>true</IsHighAotMemoryUsageTest>
   </PropertyGroup>
 

src/libraries/Microsoft.Extensions.Logging.Abstractions/tests/Microsoft.Extensions.Logging.Generators.Tests/Microsoft.Extensions.Logging.Generators.Roslyn4.0.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <RoslynApiVersion>$(MicrosoftCodeAnalysisCSharpWorkspacesVersion)</RoslynApiVersion>
+    <RoslynApiVersion>$(MicrosoftCodeAnalysisVersion_4_0)</RoslynApiVersion>
     <DefineConstants>$(DefineConstants);ROSLYN4_0_OR_GREATER</DefineConstants>
     <IsHighAotMemoryUsageTest>true</IsHighAotMemoryUsageTest>
     <EmccLinkOptimizationFlag Condition="'$(ContinuousIntegrationBuild)' == 'true'">-O1</EmccLinkOptimizationFlag>

src/libraries/System.Runtime.InteropServices/gen/DllImportGenerator/DllImportGenerator.cs

@@ -44,31 +44,14 @@ public override int GetHashCode()
             }
         }
 
-        public class IncrementalityTracker
+        public static class StepNames
         {
-            public enum StepName
-            {
-                CalculateStubInformation,
-                GenerateSingleStub,
-                NormalizeWhitespace,
-                ConcatenateStubs,
-                OutputSourceFile
-            }
-
-            public record ExecutedStepInfo(StepName Step, object Input);
-
-            private readonly List<ExecutedStepInfo> _executedSteps = new();
-            public IEnumerable<ExecutedStepInfo> ExecutedSteps => _executedSteps;
-
-            internal void RecordExecutedStep(ExecutedStepInfo step) => _executedSteps.Add(step);
+            public const string CalculateStubInformation = nameof(CalculateStubInformation);
+            public const string GenerateSingleStub = nameof(GenerateSingleStub);
+            public const string NormalizeWhitespace = nameof(NormalizeWhitespace);
+            public const string ConcatenateStubs = nameof(ConcatenateStubs);
         }
 
-        /// <summary>
-        /// This property provides a test-only hook to enable testing the incrementality of the source generator.
-        /// This will be removed when https://github.com/dotnet/roslyn/issues/54832 is implemented and can be consumed.
-        /// </summary>
-        public IncrementalityTracker? IncrementalTracker { get; set; }
-
         public void Initialize(IncrementalGeneratorInitializationContext context)
         {
             var attributedMethods = context.SyntaxProvider
@@ -120,7 +103,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
                 });
 
             IncrementalValueProvider<DllImportGeneratorOptions> stubOptions = context.AnalyzerConfigOptionsProvider
-                .Select((options, ct) => new DllImportGeneratorOptions(options.GlobalOptions));
+                .Select(static (options, ct) => new DllImportGeneratorOptions(options.GlobalOptions));
 
             IncrementalValueProvider<StubEnvironment> stubEnvironment = compilationAndTargetFramework
                 .Combine(stubOptions)
@@ -143,34 +126,24 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
                     Environment = data.Right
                 })
                 .Select(
-                    (data, ct) =>
-                    {
-                        IncrementalTracker?.RecordExecutedStep(new IncrementalityTracker.ExecutedStepInfo(IncrementalityTracker.StepName.CalculateStubInformation, data));
-                        return (data.Syntax, StubContext: CalculateStubInformation(data.Symbol, data.Environment, ct));
-                    }
+                    static (data, ct) => (data.Syntax, StubContext: CalculateStubInformation(data.Symbol, data.Environment, ct))
                 )
                 .WithComparer(Comparers.CalculatedContextWithSyntax)
+                .WithTrackingName(StepNames.CalculateStubInformation)
                 .Combine(stubOptions)
                 .Select(
-                    (data, ct) =>
-                    {
-                        IncrementalTracker?.RecordExecutedStep(new IncrementalityTracker.ExecutedStepInfo(IncrementalityTracker.StepName.GenerateSingleStub, data));
-                        return GenerateSource(data.Left.StubContext, data.Left.Syntax, data.Right);
-                    }
+                    static (data, ct) => GenerateSource(data.Left.StubContext, data.Left.Syntax, data.Right)
                 )
                 .WithComparer(Comparers.GeneratedSyntax)
+                .WithTrackingName(StepNames.GenerateSingleStub)
                 // Handle NormalizeWhitespace as a separate stage for incremental runs since it is an expensive operation.
                 .Select(
-                    (data, ct) =>
-                    {
-                        IncrementalTracker?.RecordExecutedStep(new IncrementalityTracker.ExecutedStepInfo(IncrementalityTracker.StepName.NormalizeWhitespace, data));
-                        return (data.Item1.NormalizeWhitespace().ToFullString(), data.Item2);
-                    })
+                    static (data, ct) => (data.Item1.NormalizeWhitespace().ToFullString(), data.Item2))
+                .WithTrackingName(StepNames.NormalizeWhitespace)
                 .Collect()
                 .WithComparer(Comparers.GeneratedSourceSet)
-                .Select((generatedSources, ct) =>
+                .Select(static (generatedSources, ct) =>
                 {
-                    IncrementalTracker?.RecordExecutedStep(new IncrementalityTracker.ExecutedStepInfo(IncrementalityTracker.StepName.ConcatenateStubs, generatedSources));
                     StringBuilder source = new();
                     // Mark in source that the file is auto-generated.
                     source.AppendLine("// <auto-generated/>");
@@ -182,12 +155,12 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
                     }
                     return (source: source.ToString(), diagnostics: diagnostics.ToImmutable());
                 })
-                .WithComparer(Comparers.GeneratedSource);
+                .WithComparer(Comparers.GeneratedSource)
+                .WithTrackingName(StepNames.ConcatenateStubs);
 
             context.RegisterSourceOutput(methodSourceAndDiagnostics,
-                (context, data) =>
+                static (context, data) =>
                 {
-                    IncrementalTracker?.RecordExecutedStep(new IncrementalityTracker.ExecutedStepInfo(IncrementalityTracker.StepName.OutputSourceFile, data));
                     foreach (Diagnostic diagnostic in data.Item2)
                     {
                         context.ReportDiagnostic(diagnostic);
@@ -472,7 +445,7 @@ private static IncrementalStubGenerationContext CalculateStubInformation(IMethod
             return new IncrementalStubGenerationContext(environment, dllImportStub, additionalAttributes.ToImmutableArray(), stubDllImportData, generatorDiagnostics.Diagnostics.ToImmutableArray());
         }
 
-        private (MemberDeclarationSyntax, ImmutableArray<Diagnostic>) GenerateSource(
+        private static (MemberDeclarationSyntax, ImmutableArray<Diagnostic>) GenerateSource(
             IncrementalStubGenerationContext dllImportStub,
             MethodDeclarationSyntax originalSyntax,
             DllImportGeneratorOptions options)
@@ -529,7 +502,7 @@ private static IncrementalStubGenerationContext CalculateStubInformation(IMethod
             return (PrintGeneratedSource(originalSyntax, dllImportStub.StubContext, code), dllImportStub.Diagnostics.AddRange(diagnostics.Diagnostics));
         }
 
-        private MemberDeclarationSyntax PrintForwarderStub(MethodDeclarationSyntax userDeclaredMethod, IncrementalStubGenerationContext stub, GeneratorDiagnostics diagnostics)
+        private static MemberDeclarationSyntax PrintForwarderStub(MethodDeclarationSyntax userDeclaredMethod, IncrementalStubGenerationContext stub, GeneratorDiagnostics diagnostics)
         {
             GeneratedDllImportData targetDllImportData = GetTargetDllImportDataFromStubData(
                 stub.DllImportData,