Add Fuzzlyn to CI (#60344)

Add support for Fuzzlyn in the exploratory pipeline files. * We use the pipeline name to determine which tool to use, since that seems to be the easiest way to have this available during template expansion. * All the .yml files are shared, and the setup script is also shared (renamed to fuzzer_setup.py). However the summarize and run scripts are not shared. * The summarize scripts now use the AZDO feature that allows outputting a markdown file that shows up rendered under the pipeline results. These can be seen on the "Extensions" tab of AZDO. * For Fuzzlyn, we automatically reduce silent bad codegen examples found and include these in the summary (but we do not reduce examples if we are over time). Assertion errors are not reduced, but the documentation in exploratory.md contains some information on how to reduce these manually. This should just be a temporary measure until we can more efficiently reduce these. * The issue zips are now part of the issues artifact (and I removed the "summary" part of the name) since the Fuzzlyn summarize script reads the reduced examples from the zip, and I feel it's simpler to have all the info in one artifact. Also include a small fix for the superpmi download display progress so that we do not display a larger size than the file downloaded.
dotnet · Oct 18, 2021 · 730d1f4 · 730d1f4
1 parent 14089cf
commit 730d1f4
Show file tree

Hide file tree

Showing 13 changed files with 778 additions and 207 deletions.
diff --git a/eng/pipelines/coreclr/exploratory.yml b/eng/pipelines/coreclr/exploratory.yml
@@ -1,5 +1,4 @@
-# This job definition automates the exploratory tool.
-
+# This job definition automates the fuzzing tools Antigen/Fuzzlyn
 trigger: none
 
 schedules:
@@ -10,6 +9,11 @@ schedules:
     - main
   always: true
 
+variables:
+  # Set toolName variable from pipeline name so we can use it during template expansion
+  - name: toolName
+    value: ${{ variables['Build.DefinitionName'] }}
+
 jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
@@ -46,4 +50,5 @@ jobs:
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
       testGroup: outerloop
-      liveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release
+      toolName: ${{ variables.toolName }}
diff --git a/eng/pipelines/coreclr/templates/jit-exploratory-job.yml b/eng/pipelines/coreclr/templates/jit-exploratory-job.yml
@@ -14,6 +14,7 @@ parameters:
   codeGenType: 'JIT'
   runJobTemplate: '/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml'
   additionalSetupParameters: ''
+  toolName: '' # Antigen or Fuzzlyn
 
 ### Exploratory job
 
@@ -38,6 +39,7 @@ jobs:
     testGroup: ${{ parameters.testGroup }}
     helixQueues: ${{ parameters.helixQueues }}
     additionalSetupParameters: ${{ parameters.additionalSetupParameters }}
+    toolName: ${{ parameters.toolName }}
     # Test job depends on the corresponding build job
     dependsOn:
     - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}

diff --git a/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml b/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml
@@ -18,6 +18,7 @@ parameters:
   codeGenType: 'JIT'              # optional -- Decides on the codegen technology if running on mono
   helixQueues: ''                 # required -- Helix queues
   dependOnEvaluatePaths: false
+  toolName: ''                    # required -- which tool to use: Antigen or Fuzzlyn?
 
 jobs:
 - template: xplat-pipeline-job.yml
@@ -64,12 +65,14 @@ jobs:
         value: 'py -3 -m pip'
       - name: Core_Root_Dir
         value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root'
-      - name: IssuesLocation
+      - name: HelixResults
         value: '$(Build.SourcesDirectory)\artifacts\helixresults\'
-      - name: IssuesSummaryLocation
-        value: '$(Build.SourcesDirectory)\artifacts\issues_summary\'
+      - name: IssuesLocation
+        value: '$(Build.SourcesDirectory)\artifacts\issues\'
       - name: AntigenLogsLocation
         value: '$(Build.SourcesDirectory)\artifacts\antigen_logs\'
+      - name: FuzzlynLogsLocation
+        value: '$(Build.SourcesDirectory)\artifacts\fuzzlyn_logs\'
 
     - ${{ if ne(parameters.osGroup, 'windows') }}:
       - name: PythonScript
@@ -78,13 +81,27 @@ jobs:
         value: 'pip3'
       - name: Core_Root_Dir
         value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root'
-      - name: IssuesLocation
+      - name: HelixResults
         value: '$(Build.SourcesDirectory)/artifacts/helixresults/'
-      - name: IssuesSummaryLocation
-        value: '$(Build.SourcesDirectory)/artifacts/issues_summary/'
+      - name: IssuesLocation
+        value: '$(Build.SourcesDirectory)/artifacts/issues/'
       - name: AntigenLogsLocation
         value: '$(Build.SourcesDirectory)/artifacts/antigen_logs/'
-
+      - name: FuzzlynLogsLocation
+        value: '$(Build.SourcesDirectory)/artifacts/fuzzlyn_logs/'
+
+    - ${{ if eq(parameters.toolName, 'Antigen') }}:
+      - name: SummarizeScript
+        value: 'antigen_summarize.py'
+      - name: FuzzerLogsLocation
+        value: '$(AntigenLogsLocation)'
+
+    - ${{ if eq(parameters.toolName, 'Fuzzlyn') }}:
+      - name: SummarizeScript
+        value: 'fuzzlyn_summarize.py'
+      - name: FuzzerLogsLocation
+        value: '$(FuzzlynLogsLocation)'
+
     workspace:
       clean: all
     pool:
@@ -93,8 +110,8 @@ jobs:
     steps:
     - ${{ parameters.steps }}
 
-    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/antigen_setup.py -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup)
-      displayName: ${{ format('Antigen setup ({0}-{1})', parameters.osGroup, parameters.archType) }}
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/fuzzer_setup.py -tool_name $(toolName) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup)
+      displayName: ${{ format('{0} setup ({1}-{2})', parameters.toolName, parameters.osGroup, parameters.archType) }}
 
       # Run exploratory tool in helix
     - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
@@ -112,64 +129,62 @@ jobs:
         BuildConfig: ${{ parameters.buildConfig }}
         osGroup: ${{ parameters.osGroup }}
         RunConfiguration: '$(RunConfiguration)'
-        ToolName: '$(ToolName)'
+        ToolName: ${{ parameters.toolName }}
         RunReason: '$(RunReason)'
-        continueOnError: true
-
-    - template: /eng/pipelines/common/upload-artifact-step.yml
-      parameters:
-        rootFolder: $(IssuesLocation)
-        includeRootFolder: false
-        archiveType: $(archiveType)
-        tarCompression: $(tarCompression)
-        archiveExtension: $(archiveExtension)
-        artifactName: 'Antigen_Issues_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
-        displayName: ${{ format('Upload artifacts Antigen issues for {0}-{1}', parameters.osGroup, parameters.archType ) }}
 
     # Always upload the available issues-summary.txt files
     - task: CopyFiles@2
       displayName: Copying issues-summary.txt of all partitions
       inputs:
-        sourceFolder: '$(IssuesLocation)'
+        sourceFolder: '$(HelixResults)'
         contents: '**/issues-summary-*.txt'
-        targetFolder: '$(IssuesSummaryLocation)'
+        targetFolder: '$(IssuesLocation)'
+      continueOnError: true
+      condition: always()
+
+    # Always upload the available AllIssues-*.zip files to be used for summarization
+    - task: CopyFiles@2
+      displayName: Copying AllIssues.zip of all partitions
+      inputs:
+        sourceFolder: '$(HelixResults)'
+        contents: '**/AllIssues-*.zip'
+        targetFolder: '$(IssuesLocation)'
       continueOnError: true
       condition: always()
 
     - task: PublishPipelineArtifact@1
-      displayName: Publish issues-summary.txt files
+      displayName: Publish issue files
       inputs:
-        targetPath: $(IssuesSummaryLocation)
-        artifactName: 'Issues_Summary_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+        targetPath: $(IssuesLocation)
+        artifactName: 'Issues_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
       continueOnError: true
       condition: always()
 
-    # Always upload the available Antigen.log files
+    # Always upload the available log files
     - task: CopyFiles@2
-      displayName: Copying Antigen.logs of all partitions
+      displayName: Copying fuzzer logs of all partitions
       inputs:
-        sourceFolder: '$(IssuesLocation)'
-        contents: '**/Antigen-*.log'
-        targetFolder: '$(AntigenLogsLocation)'
+        sourceFolder: '$(HelixResults)'
+        contents: '**/*.log'
+        targetFolder: '$(FuzzerLogsLocation)'
       continueOnError: true
       condition: always()
 
     - task: PublishPipelineArtifact@1
-      displayName: Publish Antigen.log files
+      displayName: Publish fuzzer log files
       inputs:
-        targetPath: $(AntigenLogsLocation)
-        artifactName: 'Antigen_Logs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+        targetPath: '$(FuzzerLogsLocation)'
+        artifactName: '$(toolName)_Logs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
       continueOnError: true
       condition: always()
 
-    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/antigen_unique_issues.py -issues_directory $(IssuesSummaryLocation)
-      displayName: ${{ format('Print unique issues ({0})', parameters.osGroup) }}
-      continueOnError: true
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/$(SummarizeScript) -issues_directory $(IssuesLocation) -arch $(archType) -platform $(osGroup)$(osSubgroup) -build_config $(buildConfig)
+      displayName: ${{ format('Summarize ({0}{1} {2})', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
       condition: always()
 
     - task: PublishPipelineArtifact@1
-      displayName: Publish Antigen build logs
+      displayName: ${{ format('Publish {0} build logs', parameters.toolName) }}
       inputs:
         targetPath: $(Build.SourcesDirectory)/artifacts/log
-        artifactName: 'Antigen_BuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+        artifactName: '$(toolName)_BuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
       condition: always()
diff --git a/src/coreclr/scripts/antigen.md b/src/coreclr/scripts/antigen.md