Add Fuzzlyn to CI

eng/pipelines/coreclr/exploratory.yml

@@ -1,5 +1,4 @@
-# This job definition automates the exploratory tool.
-
+# This job definition automates the fuzzing tools Antigen/Fuzzlyn
 trigger: none
 
 schedules:
@@ -10,6 +9,11 @@ schedules:
     - main
   always: true
 
+variables:
+  # Set toolName variable from pipeline name so we can use it during template expansion
+  - name: toolName
+    value: ${{ variables['Build.DefinitionName'] }}
+
 jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
@@ -46,4 +50,5 @@ jobs:
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
       testGroup: outerloop
-      liveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release
+      toolName: ${{ variables.toolName }}

eng/pipelines/coreclr/templates/jit-exploratory-job.yml

@@ -14,6 +14,7 @@ parameters:
   codeGenType: 'JIT'
   runJobTemplate: '/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml'
   additionalSetupParameters: ''
+  toolName: '' # Antigen or Fuzzlyn
 
 ### Exploratory job
 
@@ -38,6 +39,7 @@ jobs:
     testGroup: ${{ parameters.testGroup }}
     helixQueues: ${{ parameters.helixQueues }}
     additionalSetupParameters: ${{ parameters.additionalSetupParameters }}
+    toolName: ${{ parameters.toolName }}
     # Test job depends on the corresponding build job
     dependsOn:
     - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}

eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml

@@ -18,6 +18,7 @@ parameters:
   codeGenType: 'JIT'              # optional -- Decides on the codegen technology if running on mono
   helixQueues: ''                 # required -- Helix queues
   dependOnEvaluatePaths: false
+  toolName: ''                    # required -- which tool to use: Antigen or Fuzzlyn?
 
 jobs:
 - template: xplat-pipeline-job.yml
@@ -64,12 +65,14 @@ jobs:
         value: 'py -3 -m pip'
       - name: Core_Root_Dir
         value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root'
-      - name: IssuesLocation
+      - name: HelixResults
         value: '$(Build.SourcesDirectory)\artifacts\helixresults\'
-      - name: IssuesSummaryLocation
-        value: '$(Build.SourcesDirectory)\artifacts\issues_summary\'
+      - name: IssuesLocation
+        value: '$(Build.SourcesDirectory)\artifacts\issues\'
       - name: AntigenLogsLocation
         value: '$(Build.SourcesDirectory)\artifacts\antigen_logs\'
+      - name: FuzzlynLogsLocation
+        value: '$(Build.SourcesDirectory)\artifacts\fuzzlyn_logs\'
 
     - ${{ if ne(parameters.osGroup, 'windows') }}:
       - name: PythonScript
@@ -78,13 +81,27 @@ jobs:
         value: 'pip3'
       - name: Core_Root_Dir
         value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root'
-      - name: IssuesLocation
+      - name: HelixResults
         value: '$(Build.SourcesDirectory)/artifacts/helixresults/'
-      - name: IssuesSummaryLocation
-        value: '$(Build.SourcesDirectory)/artifacts/issues_summary/'
+      - name: IssuesLocation
+        value: '$(Build.SourcesDirectory)/artifacts/issues/'
       - name: AntigenLogsLocation
         value: '$(Build.SourcesDirectory)/artifacts/antigen_logs/'
-
+      - name: FuzzlynLogsLocation
+        value: '$(Build.SourcesDirectory)/artifacts/fuzzlyn_logs/'
+
+    - ${{ if eq(parameters.toolName, 'Antigen') }}:
+      - name: SummarizeScript
+        value: 'antigen_summarize.py'
+      - name: FuzzerLogsLocation
+        value: '$(AntigenLogsLocation)'
+
+    - ${{ if eq(parameters.toolName, 'Fuzzlyn') }}:
+      - name: SummarizeScript
+        value: 'fuzzlyn_summarize.py'
+      - name: FuzzerLogsLocation
+        value: '$(FuzzlynLogsLocation)'
+
     workspace:
       clean: all
     pool:
@@ -93,8 +110,8 @@ jobs:
     steps:
     - ${{ parameters.steps }}
 
-    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/antigen_setup.py -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup)
-      displayName: ${{ format('Antigen setup ({0}-{1})', parameters.osGroup, parameters.archType) }}
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/fuzzer_setup.py -tool_name $(toolName) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup)
+      displayName: ${{ format('{0} setup ({1}-{2})', parameters.toolName, parameters.osGroup, parameters.archType) }}
 
       # Run exploratory tool in helix
     - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
@@ -112,64 +129,62 @@ jobs:
         BuildConfig: ${{ parameters.buildConfig }}
         osGroup: ${{ parameters.osGroup }}
         RunConfiguration: '$(RunConfiguration)'
-        ToolName: '$(ToolName)'
+        ToolName: ${{ parameters.toolName }}
         RunReason: '$(RunReason)'
-        continueOnError: true
-
-    - template: /eng/pipelines/common/upload-artifact-step.yml
-      parameters:
-        rootFolder: $(IssuesLocation)
-        includeRootFolder: false
-        archiveType: $(archiveType)
-        tarCompression: $(tarCompression)
-        archiveExtension: $(archiveExtension)
-        artifactName: 'Antigen_Issues_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
-        displayName: ${{ format('Upload artifacts Antigen issues for {0}-{1}', parameters.osGroup, parameters.archType ) }}
 
     # Always upload the available issues-summary.txt files
     - task: CopyFiles@2
       displayName: Copying issues-summary.txt of all partitions
       inputs:
-        sourceFolder: '$(IssuesLocation)'
+        sourceFolder: '$(HelixResults)'
         contents: '**/issues-summary-*.txt'
-        targetFolder: '$(IssuesSummaryLocation)'
+        targetFolder: '$(IssuesLocation)'
+      continueOnError: true
+      condition: always()
+
+    # Always upload the available AllIssues-*.zip files to be used for summarization
+    - task: CopyFiles@2
+      displayName: Copying AllIssues.zip of all partitions
+      inputs:
+        sourceFolder: '$(HelixResults)'
+        contents: '**/AllIssues-*.zip'
+        targetFolder: '$(IssuesLocation)'
       continueOnError: true
       condition: always()
 
     - task: PublishPipelineArtifact@1
-      displayName: Publish issues-summary.txt files
+      displayName: Publish issue files
       inputs:
-        targetPath: $(IssuesSummaryLocation)
-        artifactName: 'Issues_Summary_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+        targetPath: $(IssuesLocation)
+        artifactName: 'Issues_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
       continueOnError: true
       condition: always()
 
-    # Always upload the available Antigen.log files
+    # Always upload the available log files
     - task: CopyFiles@2
-      displayName: Copying Antigen.logs of all partitions
+      displayName: Copying fuzzer logs of all partitions
       inputs:
-        sourceFolder: '$(IssuesLocation)'
-        contents: '**/Antigen-*.log'
-        targetFolder: '$(AntigenLogsLocation)'
+        sourceFolder: '$(HelixResults)'
+        contents: '**/*.log'
+        targetFolder: '$(FuzzerLogsLocation)'
       continueOnError: true
       condition: always()
 
     - task: PublishPipelineArtifact@1
-      displayName: Publish Antigen.log files
+      displayName: Publish fuzzer log files
       inputs:
-        targetPath: $(AntigenLogsLocation)
-        artifactName: 'Antigen_Logs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+        targetPath: '$(FuzzerLogsLocation)'
+        artifactName: '$(toolName)_Logs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
       continueOnError: true
       condition: always()
 
-    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/antigen_unique_issues.py -issues_directory $(IssuesSummaryLocation)
-      displayName: ${{ format('Print unique issues ({0})', parameters.osGroup) }}
-      continueOnError: true
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/$(SummarizeScript) -issues_directory $(IssuesLocation) -arch $(archType) -platform $(osGroup)$(osSubgroup) -build_config $(buildConfig)
+      displayName: ${{ format('Summarize ({0}{1} {2})', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
       condition: always()
 
     - task: PublishPipelineArtifact@1
-      displayName: Publish Antigen build logs
+      displayName: ${{ format('Publish {0} build logs', parameters.toolName) }}
       inputs:
         targetPath: $(Build.SourcesDirectory)/artifacts/log
-        artifactName: 'Antigen_BuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+        artifactName: '$(toolName)_BuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
       condition: always()