Add support for "known errors" to Fuzzlyn pipeline (dotnet#73938)

Fuzzlyn now supports a "known errors" list that will not be reported as failing examples and will thus not cause the pipeline to fail. The list can be specified either as a path to a file containing a JSON array of error strings, or as a built-in list "dotnet/runtime" that is maintained in Fuzzlyn itself. This PR hooks the support up on the runtime side.
jakobbotsch · Aug 17, 2022 · 1f273fd · 1f273fd
1 parent 96d1c8e
commit 1f273fd
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 6 deletions.
diff --git a/src/coreclr/scripts/fuzzlyn_run.py b/src/coreclr/scripts/fuzzlyn_run.py
@@ -202,7 +202,8 @@ def main(main_args):
                 "--seconds-to-run", str(run_duration),
                 "--output-events-to", summary_file_path,
                 "--host", path_to_corerun,
-                "--parallelism", "-1"],
+                "--parallelism", "-1",
+                "--known-errors", "dotnet/runtime"],
                 _exit_on_fail=True, _output_file=upload_fuzzer_output_path)
 
             exit_evt.set()

diff --git a/src/coreclr/scripts/fuzzlyn_summarize.py b/src/coreclr/scripts/fuzzlyn_summarize.py
@@ -135,12 +135,14 @@ def seed_from_internal_zip_path(path):
                     partition_results[partition_name]["reduced_examples"].extend(reduced_examples)
 
     total_examples_generated = 0
+    total_examples_with_known_errors = 0
     all_reduced_examples = []
     all_examples = []
     for partition_name, results in partition_results.items():
         if results['summary'] is not None:
-            # {"DegreeOfParallelism":32,"TotalProgramsGenerated":354,"TotalRunTime":"00:00:47.0918613"}
+            # {"DegreeOfParallelism":32,"TotalProgramsGenerated":354,"TotalProgramsWithKnownErrors":11,"TotalRunTime":"00:00:47.0918613"}
             total_examples_generated += results['summary']['TotalProgramsGenerated']
+            total_examples_with_known_errors += results['summary']['TotalProgramsWithKnownErrors']
 
         all_reduced_examples.extend(results['reduced_examples'])
         all_examples.extend(results['examples'])
@@ -189,6 +191,7 @@ def seed_from_internal_zip_path(path):
 
         f.write("* Total programs generated: {}\n".format(total_examples_generated))
         f.write("* Number of examples found: {}\n".format(len(all_examples)))
+        f.write("* Number of known errors hit: {}\n".format(total_examples_with_known_errors))
 
         f.write("\n")
 
@@ -225,13 +228,13 @@ def seed_from_internal_zip_path(path):
 
         if len(partition_results) > 0:
             f.write("# Run summaries per partition\n")
-            f.write("|Partition|# Programs generated|# Examples found|Run time|Degree of parallelism|\n")
-            f.write("|---|---|---|---|---|\n")
+            f.write("|Partition|# Programs generated|# Examples found|# Examples with known errors|Run time|Degree of parallelism|\n")
+            f.write("|---|---|---|---|---|---|\n")
             for partition_name, results in sorted(partition_results.items(), key=lambda p: p[0]):
                 summary = results['summary']
                 if summary is not None:
-                    # {"DegreeOfParallelism":32,"TotalProgramsGenerated":354,"TotalRunTime":"00:00:47.0918613"}
-                    f.write("|{}|{}|{}|{}|{}|\n".format(partition_name, summary['TotalProgramsGenerated'], len(results['examples']), summary['TotalRunTime'], summary['DegreeOfParallelism']))
+                    # {"DegreeOfParallelism":32,"TotalProgramsGenerated":354,"TotalProgramsWithKnownErrors":11,"TotalRunTime":"00:00:47.0918613"}
+                    f.write("|{}|{}|{}|{}|{}|{}|\n".format(partition_name, summary['TotalProgramsGenerated'], len(results['examples']), summary['TotalProgramsWithKnownErrors'], summary['TotalRunTime'], summary['DegreeOfParallelism']))
 
     print("##vso[task.uploadsummary]{}".format(md_path))