Upgrade base images, benchmark images, and Python. (#1526)

FuzzBench now works on Ubuntu 20.04 and Python 3.10.

Co-authored-by: Jonathan Metzman <[email protected]>

.github/workflows/build_and_test_run_fuzzer_benchmarks.py

@@ -92,7 +92,7 @@ def make_builds(benchmarks, fuzzer):
     # Sort benchmarks so that they get built in a deterministic order.
     fuzzer_benchmark_pairs = sorted(fuzzer_benchmark_pairs,
                                     key=lambda pair: pair[1])
-    print('Building fuzzer-benchmark pairs: {}'.format(fuzzer_benchmark_pairs))
+    print(f'Building fuzzer-benchmark pairs: {fuzzer_benchmark_pairs}')
     for _, benchmark in fuzzer_benchmark_pairs:
         make_target = get_make_target(fuzzer, benchmark)
         make_command = ['make', 'RUNNING_ON_CI=yes', '-j', make_target]
@@ -116,7 +116,7 @@ def do_build(build_type, fuzzer, always_build):
     elif build_type == 'bug':
         benchmarks = benchmark_utils.get_bug_benchmarks()
     else:
-        raise Exception('Invalid build_type: %s' % build_type)
+        raise Exception(f'Invalid build_type: {build_type}')
 
     if always_build:
         # Always do a build if always_build is True.
@@ -138,7 +138,7 @@ def do_build(build_type, fuzzer, always_build):
 def main():
     """Build OSS-Fuzz or standard benchmarks with a fuzzer."""
     if len(sys.argv) != 3:
-        print('Usage: %s <build_type> <fuzzer>' % sys.argv[0])
+        print(f'Usage: {sys.argv[0]} <build_type> <fuzzer>')
         return 1
     build_type = sys.argv[1]
     fuzzer = sys.argv[2]

.github/workflows/fuzzers.yml

@@ -17,97 +17,101 @@ jobs:
       fail-fast: false
       matrix:
         fuzzer:
+          # Default fuzzers general comparison evaluation.
           - afl
           - aflfast
-          - afl_um_prioritize
-          - afl_um_random
-          - afl_um_parallel
           - aflplusplus
-          - aflplusplus_optimal
-          - aflplusplus_tracepc
-          - aflplusplus_muttfuzz
-          - aflplusplus_um_prioritize
-          - aflplusplus_um_prioritize_75
-          - aflplusplus_um_random
-          - aflplusplus_um_random_75
-          - aflplusplus_um_random_3
-          - aflplusplus_um_random_6
-          - aflplusplus_um_parallel
           - aflsmart
           - centipede
-          - entropic
+          - eclipser
           - fairfuzz
           - honggfuzz
-          - honggfuzz_um_random
-          - honggfuzz_um_random_75
-          - honggfuzz_um_prioritize
-          - honggfuzz_um_prioritize_75
-          - honggfuzz_um_parallel
-          - lafintel
-          - klee
+          - libafl
           - libfuzzer
-          - libfuzzer_um_random
-          - libfuzzer_um_random_75
-          - libfuzzer_um_prioritize
-          - libfuzzer_um_prioritize_75
-          - libfuzzer_um_parallel
-          - libfuzzer_dataflow
-          - libfuzzer_dataflow_load
-          - libfuzzer_dataflow_store
-          - libfuzzer_dataflow_pre
           - mopt
-          - neuzz
-          - libafl
-          - libafl_text
-          - pythia_effect_bb
-          - pythia_bb
-          - fafuzz
-          - tortoisefuzz
+          # Greybox fuzzers.
           - wingfuzz
-          # Binary-only (greybox) fuzzers.
-          - eclipser
-          - eclipser_um_prioritize
-          - eclipser_um_prioritize_75
-          - eclipser_um_random
-          - eclipser_um_random_75
-          - eclipser_um_parallel
-          - afl_qemu
-          - honggfuzz_qemu
-          - weizz_qemu
-          - aflplusplus_qemu
-          - aflplusplus_frida
-          # Concolic execution
-          - fuzzolic_aflplusplus_z3
-          - fuzzolic_aflplusplus_fuzzy
-          - eclipser_aflplusplus
-          - symqemu_aflplusplus
-          - symcc_aflplusplus
-          - symcc_aflplusplus_single
-          - symcc_afl
-          - symcc_afl_single
-          # Grammar fuzzers
-          - nautilus
-          - gramatron
-          - token_level
-          - grimoire
+          # - tortoisefuzz       # To Be Fixed.
+          # Symbolic ececution.
+          # - klee               # To Be Fixed.
+          # Concolic execution.
+          # - symcc_aflplusplus  # To Be Fixed.
+          # Grammar fuzzers.
+          # - nautilus           # To Be Fixed.
+          # - gramatron          # To Be Fixed.
+          # - token_level        # To Be Fixed.
+          # - grimoire           # To Be Fixed.
           # Temporary variants.
-          - aflplusplus_dict2file
-          - afl_2_52_b
-          - aflplusplus_cmplog
-          - aflplusplus_cmplog_r
-          - aflplusplus_cmplog_1587
-          - aflplusplus_cmplog_1585
-          - afl_random_favored
-          - entropic_execute_final
-          - libfuzzer_exeute_final
-          - introspector_driven_focus
-          - libfuzzer_fork_parallel
-          - centipede_function_filter
+          # - aflplusplus_um_prioritize
+          # - aflplusplus_um_prioritize_75
+          # - aflplusplus_um_random
+          # - aflplusplus_um_random_3
+          # - aflplusplus_um_random_6
+          # - aflplusplus_um_random_75
+          # To be Removed.
+          # - introspector_driven_focus
+          # - centipede_function_filter
+          # - aflplusplus_dict2file
+          # - afl_2_52_b
+          # - aflplusplus_cmplog
+          # - afl_random_favored
+          # - entropic_execute_final
+          # - libfuzzer_exeute_final
+          # - libfuzzer_fork_parallel
+          # - afl_um_prioritize
+          # - afl_um_random
+          # - afl_um_parallel
+          # - aflplusplus_optimal
+          # - aflplusplus_tracepc
+          # - aflplusplus_um_parallel
+          # - honggfuzz_um_random
+          # - honggfuzz_um_random_75
+          # - honggfuzz_um_prioritize
+          # - honggfuzz_um_prioritize_75
+          # - honggfuzz_um_parallel
+          # - libfuzzer_um_random
+          # - libfuzzer_um_random_75
+          # - libfuzzer_um_prioritize
+          # - libfuzzer_um_prioritize_75
+          # - libfuzzer_um_parallel
+          # - libfuzzer_dataflow
+          # - libfuzzer_dataflow_load
+          # - libfuzzer_dataflow_store
+          # - libfuzzer_dataflow_pre
+          # - libafl_text
+          # - pythia_effect_bb
+          ## Binary-only (greybox) fuzzers.
+          # - eclipser_um_prioritize
+          # - eclipser_um_prioritize_75
+          # - eclipser_um_random
+          # - eclipser_um_random_75
+          # - eclipser_um_parallel
+          ## Binary-only (greybox) fuzzers.
+          # - afl_qemu
+          # - honggfuzz_qemu
+          # - weizz_qemu
+          # - aflplusplus_qemu
+          # - aflplusplus_frida
+          ## Concolic fuzzers.
+          # - fuzzolic_aflplusplus_z3
+          # - fuzzolic_aflplusplus_fuzzy
+          # - eclipser_aflplusplus
+          # - symqemu_aflplusplus
+          # - symcc_aflplusplus_single
+          # - symcc_afl
+          ## Concolic execution
+          # - symcc_afl_single
+          ## Deprecated.
+          # - entropic
+          # - lafintel
+          # - neuzz
+          # - pythia_bb
+          # - fafuzz
 
         benchmark_type:
           - oss-fuzz
           - standard
-          - bug
+          # - bug
 
     steps:
     - uses: actions/checkout@v2
@@ -126,7 +130,7 @@ jobs:
     - name: Setup Python environment
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.10.8
 
     # Copied from:
     # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions

.github/workflows/presubmit.yml

@@ -21,7 +21,7 @@ jobs:
     - name: Setup Python environment
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.10.8
 
     # Copied from:
     # https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions