IlxGen parallelization (#14372)

* IlxGen parallelization - First phase of IlxGen parallelization works sequentially and goes up to MethodBody generation, and puts the generation on a queue of work for the given file - Second phase is processing all implementation files in parallel, going trough their collected methodbody generation queues. Within a single file, the generation is sequential. - Since we now do more things in parallel, certain concurrency primitives have been removed from explicit locks in favor of thread-safe patterns, and in certain cases locking eliminated - optional env var FSHARP_EXPERIMENTAL_FEATURES support added to enable easy opt-in into new compiler features, which would otherwise have to be enabled one-by-one in fsc.exe command line settings or project files. - msbuild property -p:AdditionalFscCmdFlags added to enable passing fsc.exe CMD flags by addition and not replacement. * Adding FSHARP_EXPERIMENTAL_FEATURES regular and deterministic CI matrix runs * When running a deterministic build, parallel ilxgen is disabled (to have stable closure numeric suffixes)
dotnet · Jan 10, 2023 · b2cf133 · b2cf133
1 parent feee399
commit b2cf133
Show file tree

Hide file tree

Showing 21 changed files with 301 additions and 424 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -217,6 +217,13 @@ stages:
             name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           timeoutInMinutes: 90
+          strategy:
+            maxParallel: 2
+            matrix:
+              regular:
+                _experimental_flag: null
+              experimental_features:
+                _experimental_flag: 1
           steps:
           - checkout: self
             clean: true
@@ -229,6 +236,8 @@ stages:
               workingDirectory: $(Build.SourcesDirectory)
               installationPath: $(Build.SourcesDirectory)/.dotnet
           - script: .\eng\test-determinism.cmd -configuration Debug
+            env:
+              FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag)
             displayName: Determinism tests with Debug configuration
           - task: PublishPipelineArtifact@1
             displayName: Publish Determinism Logs
@@ -492,11 +501,22 @@ stages:
           pool:
             name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
+          strategy:
+            maxParallel: 2
+            matrix:
+              regular:
+                _experimental_flag: null
+              experimental_features:
+                _experimental_flag: 1
           steps:
           - checkout: self
             clean: true
           - script: .\Build.cmd -c Release -pack
+            env:
+              FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag)
           - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release
+            env:
+              FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag)
             displayName: End to end build tests
 
         # Up-to-date - disabled due to it being flaky
@@ -512,35 +532,8 @@ stages:
         #      filePath: eng\tests\UpToDate.ps1
         #      arguments: -configuration $(_BuildConfig) -ci -binaryLog
 
-        # Run Build with --test:ParallelCheckingWithSignatureFilesOn
-        - job: ParallelCheckingWithSignatureFiles
-          condition: eq(variables['Build.Reason'], 'PullRequest')
-          variables:
-            - name: _SignType
-              value: Test
-          pool:
-            name: $(DncEngPublicBuildPool)
-            demands: ImageOverride -equals $(WindowsMachineQueueName)
-          timeoutInMinutes: 90
-          steps:
-            - checkout: self
-              clean: true
-            - task: UseDotNet@2
-              displayName: install SDK
-              inputs:
-                packageType: sdk
-                useGlobalJson: true
-                includePreviewVersions: false
-                workingDirectory: $(Build.SourcesDirectory)
-                installationPath: $(Build.SourcesDirectory)/.dotnet
-            - script: .\build.cmd -c Release -binaryLog /p:ParallelCheckingWithSignatureFilesOn=true
-              displayName: ParallelCheckingWithSignatureFiles build with Debug configuration
-            - task: PublishPipelineArtifact@1
-              displayName: Publish ParallelCheckingWithSignatureFiles Logs
-              inputs:
-                targetPath: '$(Build.SourcesDirectory)/artifacts/log/Release'
-                artifactName: 'ParallelCheckingWithSignatureFiles Attempt $(System.JobAttempt) Logs'
-              continueOnError: true
+        # Run Build with Fsharp Experimental Features
+        # Possible change: --times:$(Build.SourcesDirectory)/artifacts/log/Release/compiler_timing.csv
 
         # Plain build Windows
         - job: Plain_Build_Windows

diff --git a/src/Compiler/AbstractIL/il.fs b/src/Compiler/AbstractIL/il.fs
@@ -3902,7 +3902,7 @@ let prependInstrsToCode (instrs: ILInstr list) (c2: ILCode) =
     // If there is a sequence point as the first instruction then keep it at the front
     | I_seqpoint _ as i0 ->
         let labels =
-            let dict = Dictionary.newWithSize c2.Labels.Count
+            let dict = Dictionary.newWithSize (c2.Labels.Count * 2) // Decrease chance of collisions by oversizing the hashtable
 
             for kvp in c2.Labels do
                 dict.Add(kvp.Key, (if kvp.Value = 0 then 0 else kvp.Value + n))
@@ -3915,7 +3915,7 @@ let prependInstrsToCode (instrs: ILInstr list) (c2: ILCode) =
         }
     | _ ->
         let labels =
-            let dict = Dictionary.newWithSize c2.Labels.Count
+            let dict = Dictionary.newWithSize (c2.Labels.Count * 2) // Decrease chance of collisions by oversizing the hashtable
 
             for kvp in c2.Labels do
                 dict.Add(kvp.Key, kvp.Value + n)

diff --git a/src/Compiler/AbstractIL/ilmorph.fs b/src/Compiler/AbstractIL/ilmorph.fs
@@ -37,7 +37,7 @@ let code_instr2instrs f (code: ILCode) =
     adjust[old] <- nw
 
     let labels =
-        let dict = Dictionary.newWithSize code.Labels.Count
+        let dict = Dictionary.newWithSize (code.Labels.Count * 2) // Decrease chance of collisions by oversizing the hashtable
 
         for kvp in code.Labels do
             dict.Add(kvp.Key, adjust[kvp.Value])

diff --git a/src/Compiler/Checking/AttributeChecking.fs b/src/Compiler/Checking/AttributeChecking.fs
@@ -522,7 +522,7 @@ let CheckRecdFieldInfoAttributes g (x:RecdFieldInfo) m =
     CheckRecdFieldAttributes g x.RecdFieldRef m
 
 // Identify any security attributes
-let IsSecurityAttribute (g: TcGlobals) amap (casmap : Dictionary<Stamp, bool>) (Attrib(tcref, _, _, _, _, _, _)) m =
+let IsSecurityAttribute (g: TcGlobals) amap (casmap : IDictionary<Stamp, bool>) (Attrib(tcref, _, _, _, _, _, _)) m =
     // There's no CAS on Silverlight, so we have to be careful here
     match g.attrib_SecurityAttribute with
     | None -> false
@@ -533,7 +533,7 @@ let IsSecurityAttribute (g: TcGlobals) amap (casmap : Dictionary<Stamp, bool>) (
             match casmap.TryGetValue tcs with
             | true, c -> c
             | _ ->
-                let exists = ExistsInEntireHierarchyOfType (fun t -> typeEquiv g t (mkAppTy attr.TyconRef [])) g amap m AllowMultiIntfInstantiations.Yes (mkAppTy tcref [])
+                let exists = ExistsInEntireHierarchyOfType (fun t -> typeEquiv g t (mkAppTy attr.TyconRef [])) g amap m AllowMultiIntfInstantiations.Yes (mkAppTy tcref [])          
                 casmap[tcs] <- exists
                 exists
         | ValueNone -> false  

diff --git a/src/Compiler/Checking/AttributeChecking.fsi b/src/Compiler/Checking/AttributeChecking.fsi
@@ -96,7 +96,7 @@ val CheckValAttributes: g: TcGlobals -> x: ValRef -> m: range -> OperationResult
 val CheckRecdFieldInfoAttributes: g: TcGlobals -> x: RecdFieldInfo -> m: range -> OperationResult<unit>
 
 val IsSecurityAttribute:
-    g: TcGlobals -> amap: Import.ImportMap -> casmap: Dictionary<Stamp, bool> -> Attrib -> m: range -> bool
+    g: TcGlobals -> amap: Import.ImportMap -> casmap: IDictionary<Stamp, bool> -> Attrib -> m: range -> bool
 
 val IsSecurityCriticalAttribute: g: TcGlobals -> Attrib -> bool
 

diff --git a/src/Compiler/Checking/MethodCalls.fs b/src/Compiler/Checking/MethodCalls.fs
@@ -1806,9 +1806,9 @@ module ProvidedMethodCalls =
              expr: Tainted<ProvidedExpr>) = 
 
         let varConv =
-            // note: using paramVars.Length as assumed initial size, but this might not 
-            // be the optimal value; this wasn't checked before obsoleting Dictionary.ofList
-            let dict = Dictionary.newWithSize paramVars.Length
+            // note: Assuming the size based on paramVars
+            // Doubling to decrease chance of collisions
+            let dict = Dictionary.newWithSize (paramVars.Length*2)
             for v, e in Seq.zip (paramVars |> Seq.map (fun x -> x.PUntaint(id, m))) (Option.toList thisArg @ allArgs) do
                 dict.Add(v, (None, e))
             dict

diff --git a/src/Compiler/Checking/QuotationTranslator.fs b/src/Compiler/Checking/QuotationTranslator.fs
@@ -22,7 +22,7 @@ open System.Collections.Generic
 
 module QP = QuotationPickler
 
-let verboseCReflect = condition "VERBOSE_CREFLECT"
+let verboseCReflect = isEnvVarSet "VERBOSE_CREFLECT"
 
 [<RequireQualifiedAccess>]
 type IsReflectedDefinition =