From 8416a57fa1d29fad266174f4468629742893501c Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Thu, 25 Aug 2022 11:24:57 -0400 Subject: [PATCH 01/18] Adding aflplusplus um modes --- .../builder.Dockerfile | 40 ++++ .../aflplusplus_um_prioritize/description.md | 14 ++ fuzzers/aflplusplus_um_prioritize/fuzzer.py | 204 ++++++++++++++++++ .../runner.Dockerfile | 23 ++ .../aflplusplus_um_random/builder.Dockerfile | 40 ++++ fuzzers/aflplusplus_um_random/description.md | 14 ++ fuzzers/aflplusplus_um_random/fuzzer.py | 180 ++++++++++++++++ .../aflplusplus_um_random/runner.Dockerfile | 23 ++ 8 files changed, 538 insertions(+) create mode 100644 fuzzers/aflplusplus_um_prioritize/builder.Dockerfile create mode 100644 fuzzers/aflplusplus_um_prioritize/description.md create mode 100755 fuzzers/aflplusplus_um_prioritize/fuzzer.py create mode 100644 fuzzers/aflplusplus_um_prioritize/runner.Dockerfile create mode 100644 fuzzers/aflplusplus_um_random/builder.Dockerfile create mode 100644 fuzzers/aflplusplus_um_random/description.md create mode 100644 fuzzers/aflplusplus_um_random/fuzzer.py create mode 100644 fuzzers/aflplusplus_um_random/runner.Dockerfile diff --git a/fuzzers/aflplusplus_um_prioritize/builder.Dockerfile b/fuzzers/aflplusplus_um_prioritize/builder.Dockerfile new file mode 100644 index 000000000..abd77021b --- /dev/null +++ b/fuzzers/aflplusplus_um_prioritize/builder.Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Install libstdc++ to use llvm_mode. +RUN apt-get update && \ + apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \ + libglib2.0-dev libpixman-1-dev python3-setuptools unzip \ + apt-utils apt-transport-https ca-certificates + +# Download and compile afl++. +RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \ + cd /afl && \ + git checkout b847e0f414e7b310e1a68bc501d4e2453bfce70e + +# Build without Python support as we don't need it. +# Set AFL_NO_X86 to skip flaky tests. +RUN cd /afl && unset CFLAGS && unset CXXFLAGS && \ + export CC=clang && export AFL_NO_X86=1 && \ + PYTHON_INCLUDE=/ make && make install && \ + make -C utils/aflpp_driver && \ + cp utils/aflpp_driver/libAFLDriver.a / + diff --git a/fuzzers/aflplusplus_um_prioritize/description.md b/fuzzers/aflplusplus_um_prioritize/description.md new file mode 100644 index 000000000..f7eb407ad --- /dev/null +++ b/fuzzers/aflplusplus_um_prioritize/description.md @@ -0,0 +1,14 @@ +# aflplusplus + +AFL++ fuzzer instance that has the following config active for all benchmarks: + - PCGUARD instrumentation + - cmplog feature + - dict2file feature + - "fast" power schedule + - persistent mode + shared memory test cases + +Repository: [https://github.com/AFLplusplus/AFLplusplus/](https://github.com/AFLplusplus/AFLplusplus/) + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/aflplusplus_um_prioritize/fuzzer.py b/fuzzers/aflplusplus_um_prioritize/fuzzer.py new file mode 100755 index 000000000..e86978121 --- /dev/null +++ b/fuzzers/aflplusplus_um_prioritize/fuzzer.py @@ -0,0 +1,204 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for AFLplusplus fuzzer.""" + +# This optimized afl++ variant should always be run together with +# "aflplusplus" to show the difference - a default configured afl++ vs. +# a hand-crafted optimized one. afl++ is configured not to enable the good +# stuff by default to be as close to vanilla afl as possible. +# But this means that the good stuff is hidden away in this benchmark +# otherwise. + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +import time +import math + +from cryptography import sys +from fuzzers.aflplusplus import fuzzer as aflplusplus_fuzzer +from fuzzers import utils + +import signal +from contextlib import contextmanager + +class TimeoutException(Exception): pass + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +PRIORITIZE_MULTIPLIER = 5 +GRACE_TIME = 3600 # 1 hour in seconds + +@contextmanager +def time_limit(seconds): + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + +def build(): # pylint: disable=too-many-branches,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + aflplusplus_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + TOTAL_FUZZING_TIME = int(os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + + SOURCE_EXTENSIONS = [".c"] #[".c", ".cc", ".cpp", ".cxx", ".h", ".hpp", ".hxx"] + NUM_MUTANTS = math.ceil((TOTAL_FUZZING_TIME * FUZZ_PROP) / DEFAULT_MUTANT_TIMEOUT) # 23 hours - half fuzzing mutants * 60 (convert to mins) * 60 (secs) / 5 mins/mutant + # Use heuristic to try to find benchmark directory, otherwise look for all files in the current directory. + subdirs = [name for name in os.listdir(src) if os.path.isdir(os.path.join(src, name))] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in SOURCE_EXTENSIONS: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", recursive=True) + + NUM_PRIORITIZED = math.ceil((NUM_MUTANTS * PRIORITIZE_MULTIPLIER) / len(source_files)) + print(NUM_PRIORITIZED) + + prioritize_map = {} + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob(f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants = [f"{source_dir}/{mutant.split('/')[-1]}"[1:] for mutant in mutants_glob] + with open(f"{mutate_dir}/mutants.txt", "w") as f: + f.writelines("%s\n" % l for l in mutants) + os.system(f"prioritize_mutants {mutate_dir}/mutants.txt {mutate_dir}/prioritize_mutants_sorted.txt {NUM_PRIORITIZED} --noSDPriority --sourceDir {src} --mutantDir {mutate_dir}") + prioritized_list = [] + with open(f"{mutate_dir}/prioritize_mutants_sorted.txt", "r") as f: + prioritized_list = f.read().splitlines() + prioritize_map[source_file] = prioritized_list + + prioritized_keys = list(prioritize_map.keys()) + random.shuffle(prioritized_keys) + order = [] + ind = 0 + finished = False + + while not finished: + finished = True + for key in prioritized_keys: + if ind < len(prioritize_map[key]): + finished = False + order.append((key, ind)) + ind += 1 + print(order) + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while num_non_buggy <= NUM_MUTANTS and ind < len(order): + with utils.restore_directory(src), utils.restore_directory(work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}.{num_non_buggy}" + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', f'{out}/{orig_fuzz_target}', shallow=False): + print(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except Exception as e: + print(e) + print(f"EXECEPTION {num_non_buggy}, ind: {ind}") + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException as e: + print(e) + pass + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + TOTAL_FUZZING_TIME = int(os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + TOTAL_MUTANT_TIME = int(FUZZ_PROP * TOTAL_FUZZING_TIME) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + TIMEOUT = int(TOTAL_MUTANT_TIME / max(len(mutants), 1)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + os.environ['AFL_SKIP_CRASHES'] = "1" + + for mutant in mutants: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(TIMEOUT): + aflplusplus_fuzzer.fuzz(input_corpus, + output_corpus, + mutant) + except TimeoutException as e: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) \ No newline at end of file diff --git a/fuzzers/aflplusplus_um_prioritize/runner.Dockerfile b/fuzzers/aflplusplus_um_prioritize/runner.Dockerfile new file mode 100644 index 000000000..7aa1da8e4 --- /dev/null +++ b/fuzzers/aflplusplus_um_prioritize/runner.Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# This makes interactive docker runs painless: +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out" +#ENV AFL_MAP_SIZE=2621440 +ENV PATH="$PATH:/out" +ENV AFL_SKIP_CPUFREQ=1 +ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 +ENV AFL_TESTCACHE_SIZE=2 diff --git a/fuzzers/aflplusplus_um_random/builder.Dockerfile b/fuzzers/aflplusplus_um_random/builder.Dockerfile new file mode 100644 index 000000000..abd77021b --- /dev/null +++ b/fuzzers/aflplusplus_um_random/builder.Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Install libstdc++ to use llvm_mode. +RUN apt-get update && \ + apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \ + libglib2.0-dev libpixman-1-dev python3-setuptools unzip \ + apt-utils apt-transport-https ca-certificates + +# Download and compile afl++. +RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \ + cd /afl && \ + git checkout b847e0f414e7b310e1a68bc501d4e2453bfce70e + +# Build without Python support as we don't need it. +# Set AFL_NO_X86 to skip flaky tests. +RUN cd /afl && unset CFLAGS && unset CXXFLAGS && \ + export CC=clang && export AFL_NO_X86=1 && \ + PYTHON_INCLUDE=/ make && make install && \ + make -C utils/aflpp_driver && \ + cp utils/aflpp_driver/libAFLDriver.a / + diff --git a/fuzzers/aflplusplus_um_random/description.md b/fuzzers/aflplusplus_um_random/description.md new file mode 100644 index 000000000..f7eb407ad --- /dev/null +++ b/fuzzers/aflplusplus_um_random/description.md @@ -0,0 +1,14 @@ +# aflplusplus + +AFL++ fuzzer instance that has the following config active for all benchmarks: + - PCGUARD instrumentation + - cmplog feature + - dict2file feature + - "fast" power schedule + - persistent mode + shared memory test cases + +Repository: [https://github.com/AFLplusplus/AFLplusplus/](https://github.com/AFLplusplus/AFLplusplus/) + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/aflplusplus_um_random/fuzzer.py b/fuzzers/aflplusplus_um_random/fuzzer.py new file mode 100644 index 000000000..3079d42f4 --- /dev/null +++ b/fuzzers/aflplusplus_um_random/fuzzer.py @@ -0,0 +1,180 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for AFLplusplus fuzzer.""" + +# This optimized afl++ variant should always be run together with +# "aflplusplus" to show the difference - a default configured afl++ vs. +# a hand-crafted optimized one. afl++ is configured not to enable the good +# stuff by default to be as close to vanilla afl as possible. +# But this means that the good stuff is hidden away in this benchmark +# otherwise. + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +import time + +from cryptography import sys +from fuzzers.aflplusplus import fuzzer as aflplusplus_fuzzer +from fuzzers import utils + +import signal +from contextlib import contextmanager + +class TimeoutException(Exception): pass + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +PRIORITIZE_MULTIPLIER = 3 +GRACE_TIME = 3600 # 1 hour in seconds + +@contextmanager +def time_limit(seconds): + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + +def build(): # pylint: disable=too-many-branches,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + aflplusplus_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + TOTAL_FUZZING_TIME = int(os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + + SOURCE_EXTENSIONS = [".c"] #[".c", ".cc", ".cpp", ".cxx", ".h", ".hpp", ".hxx"] + NUM_MUTANTS = int((TOTAL_FUZZING_TIME * FUZZ_PROP) / DEFAULT_MUTANT_TIMEOUT) # 23 hours - half fuzzing mutants * 60 (convert to mins) * 60 (secs) / 5 mins/mutant + # Use heuristic to try to find benchmark directory, otherwise look for all files in the current directory. + subdirs = [name for name in os.listdir(src) if os.path.isdir(os.path.join(src, name))] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in SOURCE_EXTENSIONS: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", recursive=True) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob(f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [f"{source_dir}/{mutant.split('/')[-1]}"[1:] for mutant in mutants_glob] + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w") as f: + f.writelines("%s\n" % l for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while num_non_buggy <= NUM_MUTANTS and ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory(work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}.{num_non_buggy}" + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', f'{out}/{orig_fuzz_target}', shallow=False): + print(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except Exception as e: + print(e) + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException as e: + print(e) + pass + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + TOTAL_FUZZING_TIME = int(os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + TOTAL_MUTANT_TIME = int(FUZZ_PROP * TOTAL_FUZZING_TIME) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + TIMEOUT = int(TOTAL_MUTANT_TIME / max(len(mutants), 1)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + os.environ['AFL_SKIP_CRASHES'] = "1" + + for mutant in mutants: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(TIMEOUT): + aflplusplus_fuzzer.fuzz(input_corpus, + output_corpus, + mutant) + except TimeoutException as e: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) \ No newline at end of file diff --git a/fuzzers/aflplusplus_um_random/runner.Dockerfile b/fuzzers/aflplusplus_um_random/runner.Dockerfile new file mode 100644 index 000000000..7aa1da8e4 --- /dev/null +++ b/fuzzers/aflplusplus_um_random/runner.Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# This makes interactive docker runs painless: +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out" +#ENV AFL_MAP_SIZE=2621440 +ENV PATH="$PATH:/out" +ENV AFL_SKIP_CPUFREQ=1 +ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 +ENV AFL_TESTCACHE_SIZE=2 From c230e1afd95bbfefc3ed350d8ee510b2102c333c Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Thu, 25 Aug 2022 20:39:37 -0400 Subject: [PATCH 02/18] Adding fuzzers to fuzzers.yml and creating an experiment --- .github/workflows/fuzzers.yml | 2 + fuzzers/aflplusplus_um_prioritize/fuzzer.py | 138 ++++++++++++-------- fuzzers/aflplusplus_um_random/fuzzer.py | 115 +++++++++------- service/experiment-requests.yaml | 7 + 4 files changed, 160 insertions(+), 102 deletions(-) diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml index 702e76e8b..b3b373b8b 100644 --- a/.github/workflows/fuzzers.yml +++ b/.github/workflows/fuzzers.yml @@ -24,6 +24,8 @@ jobs: - aflplusplus_optimal - aflplusplus_tracepc - aflplusplus_zafl + - aflplusplus_um_prioritize + - aflplusplus_um_random - aflsmart - centipede - entropic diff --git a/fuzzers/aflplusplus_um_prioritize/fuzzer.py b/fuzzers/aflplusplus_um_prioritize/fuzzer.py index e86978121..7b5d3831c 100755 --- a/fuzzers/aflplusplus_um_prioritize/fuzzer.py +++ b/fuzzers/aflplusplus_um_prioritize/fuzzer.py @@ -28,27 +28,32 @@ import filecmp import time import math +import signal +from contextlib import contextmanager -from cryptography import sys from fuzzers.aflplusplus import fuzzer as aflplusplus_fuzzer from fuzzers import utils -import signal -from contextlib import contextmanager -class TimeoutException(Exception): pass +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" -TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours -TOTAL_BUILD_TIME = 43200 # 12 hours + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours FUZZ_PROP = 0.5 DEFAULT_MUTANT_TIMEOUT = 300 -PRIORITIZE_MULTIPLIER = 5 -GRACE_TIME = 3600 # 1 hour in seconds +PRIORITIZE_MULTIPLIER = 20 +GRACE_TIME = 3600 # 1 hour in seconds + @contextmanager def time_limit(seconds): + """Method to define a time limit before throwing exception""" + def signal_handler(signum, frame): raise TimeoutException("Timed out!") + signal.signal(signal.SIGALRM, signal_handler) signal.alarm(seconds) try: @@ -56,7 +61,8 @@ def signal_handler(signum, frame): finally: signal.alarm(0) -def build(): # pylint: disable=too-many-branches,too-many-statements + +def build(): # pylint: disable=too-many-locals,too-many-statements """Build benchmark.""" start_time = time.time() @@ -75,14 +81,21 @@ def build(): # pylint: disable=too-many-branches,too-many-statements orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): aflplusplus_fuzzer.build() - shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") benchmark = os.getenv("BENCHMARK") - TOTAL_FUZZING_TIME = int(os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) - - SOURCE_EXTENSIONS = [".c"] #[".c", ".cc", ".cpp", ".cxx", ".h", ".hpp", ".hxx"] - NUM_MUTANTS = math.ceil((TOTAL_FUZZING_TIME * FUZZ_PROP) / DEFAULT_MUTANT_TIMEOUT) # 23 hours - half fuzzing mutants * 60 (convert to mins) * 60 (secs) / 5 mins/mutant - # Use heuristic to try to find benchmark directory, otherwise look for all files in the current directory. - subdirs = [name for name in os.listdir(src) if os.path.isdir(os.path.join(src, name))] + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + + source_extensions = [".c"] + num_mutants = math.ceil( + (total_fuzzing_time * FUZZ_PROP) / DEFAULT_MUTANT_TIMEOUT) + # Use heuristic to try to find benchmark directory, otherwise look for all + # files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] benchmark_src_dir = src for directory in subdirs: if directory in benchmark: @@ -90,26 +103,36 @@ def build(): # pylint: disable=too-many-branches,too-many-statements break source_files = [] - for extension in SOURCE_EXTENSIONS: - source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", recursive=True) + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) - NUM_PRIORITIZED = math.ceil((NUM_MUTANTS * PRIORITIZE_MULTIPLIER) / len(source_files)) - print(NUM_PRIORITIZED) + num_prioritized = math.ceil( + (num_mutants * PRIORITIZE_MULTIPLIER) / len(source_files)) prioritize_map = {} for source_file in source_files: source_dir = os.path.dirname(source_file).split(src, 1)[1] Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) - os.system(f"mutate {source_file} --mutantDir {mutate_dir}/{source_dir} --noCheck > /dev/null") + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") source_base = os.path.basename(source_file).split(".")[0] - mutants_glob = glob.glob(f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") - mutants = [f"{source_dir}/{mutant.split('/')[-1]}"[1:] for mutant in mutants_glob] - with open(f"{mutate_dir}/mutants.txt", "w") as f: - f.writelines("%s\n" % l for l in mutants) - os.system(f"prioritize_mutants {mutate_dir}/mutants.txt {mutate_dir}/prioritize_mutants_sorted.txt {NUM_PRIORITIZED} --noSDPriority --sourceDir {src} --mutantDir {mutate_dir}") + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants = [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf_8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + os.system(f"prioritize_mutants {mutate_dir}/mutants.txt \ + {mutate_dir}/prioritize_mutants_sorted.txt {num_prioritized}\ + --noSDPriority --sourceDir {src} --mutantDir {mutate_dir}") prioritized_list = [] - with open(f"{mutate_dir}/prioritize_mutants_sorted.txt", "r") as f: - prioritized_list = f.read().splitlines() + with open(f"{mutate_dir}/prioritize_mutants_sorted.txt", + "r", + encoding="utf_8") as f_name: + prioritized_list = f_name.read().splitlines() prioritize_map[source_file] = prioritized_list prioritized_keys = list(prioritize_map.keys()) @@ -129,16 +152,18 @@ def build(): # pylint: disable=too-many-branches,too-many-statements curr_time = time.time() # Add grace time for final build at end - remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) try: with time_limit(remaining_time): num_non_buggy = 1 ind = 0 - while num_non_buggy <= NUM_MUTANTS and ind < len(order): - with utils.restore_directory(src), utils.restore_directory(work): + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): key, line = order[ind] - mutant = prioritize_map[key][line] + mutant = prioritize_map[key][line] print(mutant) suffix = "." + mutant.split(".")[-1] mpart = ".mutant." + mutant.split(".mutant.")[1] @@ -148,57 +173,60 @@ def build(): # pylint: disable=too-many-branches,too-many-statements os.system(f"cp {source_file} {mutate_dir}/orig") os.system(f"cp {mutate_dir}/{mutant} {source_file}") try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}.{num_non_buggy}" + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + os.system(f"rm -rf {out}/*") aflplusplus_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', f'{out}/{orig_fuzz_target}', shallow=False): - print(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{new_fuzz_target}") + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") num_non_buggy += 1 - print(f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") + print( + f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") else: print(f"EQUAL {num_non_buggy}, ind: {ind}") - except Exception as e: - print(e) - print(f"EXECEPTION {num_non_buggy}, ind: {ind}") + except RuntimeError: pass os.system(f"cp {mutate_dir}/orig {source_file}") ind += 1 - except TimeoutException as e: - print(e) + except TimeoutException: pass - + os.system(f"rm -rf {out}/*") aflplusplus_fuzzer.build() os.system(f"cp {mutate_bins}/* {out}/") - - def fuzz(input_corpus, output_corpus, target_binary): """Run fuzzer.""" - TOTAL_FUZZING_TIME = int(os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) - TOTAL_MUTANT_TIME = int(FUZZ_PROP * TOTAL_FUZZING_TIME) + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) mutants = glob.glob(f"{target_binary}.*") random.shuffle(mutants) - TIMEOUT = int(TOTAL_MUTANT_TIME / max(len(mutants), 1)) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" os.mkdir("/storage") os.mkdir(input_corpus_dir) os.environ['AFL_SKIP_CRASHES'] = "1" - for mutant in mutants: + for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") try: - with time_limit(TIMEOUT): - aflplusplus_fuzzer.fuzz(input_corpus, - output_corpus, - mutant) - except TimeoutException as e: + with time_limit(timeout): + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: pass os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) \ No newline at end of file + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/aflplusplus_um_random/fuzzer.py b/fuzzers/aflplusplus_um_random/fuzzer.py index 3079d42f4..214452e5d 100644 --- a/fuzzers/aflplusplus_um_random/fuzzer.py +++ b/fuzzers/aflplusplus_um_random/fuzzer.py @@ -27,27 +27,33 @@ import shutil import filecmp import time +import signal +import math +from contextlib import contextmanager -from cryptography import sys from fuzzers.aflplusplus import fuzzer as aflplusplus_fuzzer from fuzzers import utils -import signal -from contextlib import contextmanager -class TimeoutException(Exception): pass +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" -TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours -TOTAL_BUILD_TIME = 43200 # 12 hours + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours FUZZ_PROP = 0.5 DEFAULT_MUTANT_TIMEOUT = 300 PRIORITIZE_MULTIPLIER = 3 -GRACE_TIME = 3600 # 1 hour in seconds +GRACE_TIME = 3600 # 1 hour in seconds + @contextmanager def time_limit(seconds): + """Method to define a time limit before throwing exception""" + def signal_handler(signum, frame): raise TimeoutException("Timed out!") + signal.signal(signal.SIGALRM, signal_handler) signal.alarm(seconds) try: @@ -55,7 +61,8 @@ def signal_handler(signum, frame): finally: signal.alarm(0) -def build(): # pylint: disable=too-many-branches,too-many-statements + +def build(): # pylint: disable=too-many-locals,too-many-statements """Build benchmark.""" start_time = time.time() @@ -74,14 +81,17 @@ def build(): # pylint: disable=too-many-branches,too-many-statements orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): aflplusplus_fuzzer.build() - shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") benchmark = os.getenv("BENCHMARK") - TOTAL_FUZZING_TIME = int(os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) - SOURCE_EXTENSIONS = [".c"] #[".c", ".cc", ".cpp", ".cxx", ".h", ".hpp", ".hxx"] - NUM_MUTANTS = int((TOTAL_FUZZING_TIME * FUZZ_PROP) / DEFAULT_MUTANT_TIMEOUT) # 23 hours - half fuzzing mutants * 60 (convert to mins) * 60 (secs) / 5 mins/mutant - # Use heuristic to try to find benchmark directory, otherwise look for all files in the current directory. - subdirs = [name for name in os.listdir(src) if os.path.isdir(os.path.join(src, name))] + source_extensions = [".c"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] benchmark_src_dir = src for directory in subdirs: if directory in benchmark: @@ -89,34 +99,42 @@ def build(): # pylint: disable=too-many-branches,too-many-statements break source_files = [] - for extension in SOURCE_EXTENSIONS: - source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", recursive=True) + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) mutants = [] for source_file in source_files: source_dir = os.path.dirname(source_file).split(src, 1)[1] Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) - os.system(f"mutate {source_file} --mutantDir {mutate_dir}/{source_dir} --noCheck > /dev/null") + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") source_base = os.path.basename(source_file).split(".")[0] - mutants_glob = glob.glob(f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") - mutants += [f"{source_dir}/{mutant.split('/')[-1]}"[1:] for mutant in mutants_glob] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] - random.shuffle(mutants) - with open(f"{mutate_dir}/mutants.txt", "w") as f: - f.writelines("%s\n" % l for l in mutants) + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) curr_time = time.time() # Add grace time for final build at end - remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) try: with time_limit(remaining_time): num_non_buggy = 1 ind = 0 - while num_non_buggy <= NUM_MUTANTS and ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory(work): - mutant = mutants[ind] + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] suffix = "." + mutant.split(".")[-1] mpart = ".mutant." + mutant.split(".mutant.")[1] source_file = f"{src}/{mutant.replace(mpart, suffix)}" @@ -126,55 +144,58 @@ def build(): # pylint: disable=too-many-branches,too-many-statements os.system(f"cp {mutate_dir}/{mutant} {source_file}") try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}.{num_non_buggy}" + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + os.system(f"rm -rf {out}/*") aflplusplus_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', f'{out}/{orig_fuzz_target}', shallow=False): - print(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{new_fuzz_target}") + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") num_non_buggy += 1 else: print("EQUAL") - except Exception as e: - print(e) + except RuntimeError: pass os.system(f"cp {mutate_dir}/orig {source_file}") ind += 1 - except TimeoutException as e: - print(e) + except TimeoutException: pass - + os.system(f"rm -rf {out}/*") aflplusplus_fuzzer.build() os.system(f"cp {mutate_bins}/* {out}/") - - def fuzz(input_corpus, output_corpus, target_binary): """Run fuzzer.""" - TOTAL_FUZZING_TIME = int(os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) - TOTAL_MUTANT_TIME = int(FUZZ_PROP * TOTAL_FUZZING_TIME) + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) mutants = glob.glob(f"{target_binary}.*") random.shuffle(mutants) - TIMEOUT = int(TOTAL_MUTANT_TIME / max(len(mutants), 1)) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" os.mkdir("/storage") os.mkdir(input_corpus_dir) os.environ['AFL_SKIP_CRASHES'] = "1" - for mutant in mutants: + for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") try: - with time_limit(TIMEOUT): - aflplusplus_fuzzer.fuzz(input_corpus, - output_corpus, - mutant) - except TimeoutException as e: + with time_limit(timeout): + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: pass os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) \ No newline at end of file + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/service/experiment-requests.yaml b/service/experiment-requests.yaml index fced02c46..23cdd909d 100644 --- a/service/experiment-requests.yaml +++ b/service/experiment-requests.yaml @@ -20,6 +20,13 @@ # Please add new experiment requests towards the top of this file. # +- experiment: 2022-08-25-aflpp-um + description: "afl++ um coverage experiments (compare our fuzzers against afl++)" + fuzzers: + - aflplusplus_um_prioritize + - aflplusplus_um_random + - aflplusplus + - experiment: 2022-08-18-libfuzzer-focus description: "Testing with focus function driven by fuzz introspector results" fuzzers: From 1657e164c1eea93a292eb4273f98137f729cc0f1 Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Wed, 31 Aug 2022 10:36:16 -0400 Subject: [PATCH 03/18] Adding UM fuzzers for honggfuzz, libfuzzer, eclipser, and afl --- .github/workflows/fuzzers.yml | 8 + .../go-ethereum_fuzzvmruntime/benchmark.yaml | 3 + benchmarks/libxslt_xpath/benchmark.yaml | 2 + fuzzers/afl_um_prioritize/builder.Dockerfile | 36 +++ fuzzers/afl_um_prioritize/description.md | 9 + fuzzers/afl_um_prioritize/fuzzer.py | 228 ++++++++++++++++++ fuzzers/afl_um_prioritize/runner.Dockerfile | 15 ++ fuzzers/afl_um_random/builder.Dockerfile | 36 +++ fuzzers/afl_um_random/description.md | 10 + fuzzers/afl_um_random/fuzzer.py | 196 +++++++++++++++ fuzzers/afl_um_random/runner.Dockerfile | 15 ++ .../aflplusplus_um_prioritize/description.md | 11 +- fuzzers/aflplusplus_um_prioritize/fuzzer.py | 11 +- fuzzers/aflplusplus_um_random/description.md | 12 +- fuzzers/aflplusplus_um_random/fuzzer.py | 10 +- .../eclipser_um_prioritize/builder.Dockerfile | 43 ++++ fuzzers/eclipser_um_prioritize/description.md | 9 + fuzzers/eclipser_um_prioritize/fuzzer.py | 227 +++++++++++++++++ .../eclipser_um_prioritize/runner.Dockerfile | 51 ++++ fuzzers/eclipser_um_random/builder.Dockerfile | 43 ++++ fuzzers/eclipser_um_random/description.md | 10 + fuzzers/eclipser_um_random/fuzzer.py | 196 +++++++++++++++ fuzzers/eclipser_um_random/runner.Dockerfile | 51 ++++ .../builder.Dockerfile | 40 +++ .../honggfuzz_um_prioritize/description.md | 9 + fuzzers/honggfuzz_um_prioritize/fuzzer.py | 227 +++++++++++++++++ .../honggfuzz_um_prioritize/runner.Dockerfile | 18 ++ .../honggfuzz_um_random/builder.Dockerfile | 40 +++ fuzzers/honggfuzz_um_random/description.md | 10 + fuzzers/honggfuzz_um_random/fuzzer.py | 196 +++++++++++++++ fuzzers/honggfuzz_um_random/runner.Dockerfile | 18 ++ .../builder.Dockerfile | 29 +++ .../libfuzzer_um_prioritize/description.md | 9 + fuzzers/libfuzzer_um_prioritize/fuzzer.py | 224 +++++++++++++++++ .../libfuzzer_um_prioritize/runner.Dockerfile | 15 ++ .../libfuzzer_um_random/builder.Dockerfile | 29 +++ fuzzers/libfuzzer_um_random/description.md | 9 + fuzzers/libfuzzer_um_random/fuzzer.py | 192 +++++++++++++++ fuzzers/libfuzzer_um_random/runner.Dockerfile | 15 ++ service/experiment-requests.yaml | 6 + 40 files changed, 2294 insertions(+), 24 deletions(-) create mode 100644 fuzzers/afl_um_prioritize/builder.Dockerfile create mode 100644 fuzzers/afl_um_prioritize/description.md create mode 100755 fuzzers/afl_um_prioritize/fuzzer.py create mode 100644 fuzzers/afl_um_prioritize/runner.Dockerfile create mode 100644 fuzzers/afl_um_random/builder.Dockerfile create mode 100644 fuzzers/afl_um_random/description.md create mode 100644 fuzzers/afl_um_random/fuzzer.py create mode 100644 fuzzers/afl_um_random/runner.Dockerfile create mode 100644 fuzzers/eclipser_um_prioritize/builder.Dockerfile create mode 100644 fuzzers/eclipser_um_prioritize/description.md create mode 100755 fuzzers/eclipser_um_prioritize/fuzzer.py create mode 100644 fuzzers/eclipser_um_prioritize/runner.Dockerfile create mode 100644 fuzzers/eclipser_um_random/builder.Dockerfile create mode 100644 fuzzers/eclipser_um_random/description.md create mode 100644 fuzzers/eclipser_um_random/fuzzer.py create mode 100644 fuzzers/eclipser_um_random/runner.Dockerfile create mode 100644 fuzzers/honggfuzz_um_prioritize/builder.Dockerfile create mode 100644 fuzzers/honggfuzz_um_prioritize/description.md create mode 100755 fuzzers/honggfuzz_um_prioritize/fuzzer.py create mode 100644 fuzzers/honggfuzz_um_prioritize/runner.Dockerfile create mode 100644 fuzzers/honggfuzz_um_random/builder.Dockerfile create mode 100644 fuzzers/honggfuzz_um_random/description.md create mode 100644 fuzzers/honggfuzz_um_random/fuzzer.py create mode 100644 fuzzers/honggfuzz_um_random/runner.Dockerfile create mode 100644 fuzzers/libfuzzer_um_prioritize/builder.Dockerfile create mode 100644 fuzzers/libfuzzer_um_prioritize/description.md create mode 100755 fuzzers/libfuzzer_um_prioritize/fuzzer.py create mode 100644 fuzzers/libfuzzer_um_prioritize/runner.Dockerfile create mode 100644 fuzzers/libfuzzer_um_random/builder.Dockerfile create mode 100644 fuzzers/libfuzzer_um_random/description.md create mode 100755 fuzzers/libfuzzer_um_random/fuzzer.py create mode 100644 fuzzers/libfuzzer_um_random/runner.Dockerfile diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml index b3b373b8b..cc0206e47 100644 --- a/.github/workflows/fuzzers.yml +++ b/.github/workflows/fuzzers.yml @@ -20,6 +20,8 @@ jobs: - afl - aflcc - aflfast + - afl_um_prioritize + - afl_um_random - aflplusplus - aflplusplus_optimal - aflplusplus_tracepc @@ -31,9 +33,13 @@ jobs: - entropic - fairfuzz - honggfuzz + - honggfuzz_um_random + - honggfuzz_um_prioritize - lafintel - klee - libfuzzer + - libfuzzer_um_random + - libfuzzer_um_prioritize - libfuzzer_dataflow - libfuzzer_dataflow_load - libfuzzer_dataflow_store @@ -47,6 +53,8 @@ jobs: - tortoisefuzz # Binary-only (greybox) fuzzers. - eclipser + - eclipser_um_prioritize + - eclipser_um_random - afl_qemu - honggfuzz_qemu - weizz_qemu diff --git a/benchmarks/go-ethereum_fuzzvmruntime/benchmark.yaml b/benchmarks/go-ethereum_fuzzvmruntime/benchmark.yaml index f33ab09fb..be935c162 100644 --- a/benchmarks/go-ethereum_fuzzvmruntime/benchmark.yaml +++ b/benchmarks/go-ethereum_fuzzvmruntime/benchmark.yaml @@ -3,3 +3,6 @@ commit_date: 2021-07-01 07:00:00+00:00 fuzz_target: fuzzVmRuntime project: go-ethereum language: go +unsupported_fuzzers: + - aflplusplus_um_random + - afplusplus_um_prioritize diff --git a/benchmarks/libxslt_xpath/benchmark.yaml b/benchmarks/libxslt_xpath/benchmark.yaml index 9f761d564..7f188c85a 100644 --- a/benchmarks/libxslt_xpath/benchmark.yaml +++ b/benchmarks/libxslt_xpath/benchmark.yaml @@ -5,6 +5,8 @@ project: libxslt unsupported_fuzzers: - aflcc - aflplusplus_qemu + - aflplusplus_um_prioritize + - aflplusplus_um_random - aflplusplus_qemu_tracepc - aflplusplus_frida - klee diff --git a/fuzzers/afl_um_prioritize/builder.Dockerfile b/fuzzers/afl_um_prioritize/builder.Dockerfile new file mode 100644 index 000000000..253ac9f87 --- /dev/null +++ b/fuzzers/afl_um_prioritize/builder.Dockerfile @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Download and compile AFL v2.57b. +# Set AFL_NO_X86 to skip flaky tests. +RUN git clone https://github.com/google/AFL.git /afl && \ + cd /afl && \ + git checkout 61037103ae3722c8060ff7082994836a794f978e && \ + AFL_NO_X86=1 make + +# Use afl_driver.cpp from LLVM as our fuzzing library. +RUN apt-get update && \ + apt-get install wget -y && \ + wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \ + clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \ + clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \ + ar r /libAFL.a *.o + diff --git a/fuzzers/afl_um_prioritize/description.md b/fuzzers/afl_um_prioritize/description.md new file mode 100644 index 000000000..8eade8ff9 --- /dev/null +++ b/fuzzers/afl_um_prioritize/description.md @@ -0,0 +1,9 @@ +# afl UM (prioritize) + +Run afl over mutated code with UM prioritization + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/afl_um_prioritize/fuzzer.py b/fuzzers/afl_um_prioritize/fuzzer.py new file mode 100755 index 000000000..8a22b20dd --- /dev/null +++ b/fuzzers/afl_um_prioritize/fuzzer.py @@ -0,0 +1,228 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for afl fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import math +import signal +from contextlib import contextmanager + +from fuzzers.afl import fuzzer as afl_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +PRIORITIZE_MULTIPLIER = 20 +GRACE_TIME = 3600 # 1 hour in seconds + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-branches + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + afl_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + + source_extensions = [".c", ".cc", ".cpp"] + num_mutants = math.ceil( + (total_fuzzing_time * FUZZ_PROP) / DEFAULT_MUTANT_TIMEOUT) + # Use heuristic to try to find benchmark directory, otherwise look for all + # files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + + num_prioritized = math.ceil( + (num_mutants * PRIORITIZE_MULTIPLIER) / len(source_files)) + + prioritize_map = {} + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants = [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf_8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + os.system(f"prioritize_mutants {mutate_dir}/mutants.txt \ + {mutate_dir}/prioritize_mutants_sorted.txt {num_prioritized}\ + --noSDPriority --sourceDir {src} --mutantDir {mutate_dir}") + prioritized_list = [] + with open(f"{mutate_dir}/prioritize_mutants_sorted.txt", + "r", + encoding="utf_8") as f_name: + prioritized_list = f_name.read().splitlines() + prioritize_map[source_file] = prioritized_list + + prioritized_keys = list(prioritize_map.keys()) + random.shuffle(prioritized_keys) + order = [] + ind = 0 + finished = False + + while not finished: + finished = True + for key in prioritized_keys: + if ind < len(prioritize_map[key]): + finished = False + order.append((key, ind)) + ind += 1 + print(order) + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print( + f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + os.environ['AFL_SKIP_CRASHES'] = "1" + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + afl_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + afl_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/afl_um_prioritize/runner.Dockerfile b/fuzzers/afl_um_prioritize/runner.Dockerfile new file mode 100644 index 000000000..0d6cf004e --- /dev/null +++ b/fuzzers/afl_um_prioritize/runner.Dockerfile @@ -0,0 +1,15 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image diff --git a/fuzzers/afl_um_random/builder.Dockerfile b/fuzzers/afl_um_random/builder.Dockerfile new file mode 100644 index 000000000..253ac9f87 --- /dev/null +++ b/fuzzers/afl_um_random/builder.Dockerfile @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Download and compile AFL v2.57b. +# Set AFL_NO_X86 to skip flaky tests. +RUN git clone https://github.com/google/AFL.git /afl && \ + cd /afl && \ + git checkout 61037103ae3722c8060ff7082994836a794f978e && \ + AFL_NO_X86=1 make + +# Use afl_driver.cpp from LLVM as our fuzzing library. +RUN apt-get update && \ + apt-get install wget -y && \ + wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \ + clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \ + clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \ + ar r /libAFL.a *.o + diff --git a/fuzzers/afl_um_random/description.md b/fuzzers/afl_um_random/description.md new file mode 100644 index 000000000..f0c6fed56 --- /dev/null +++ b/fuzzers/afl_um_random/description.md @@ -0,0 +1,10 @@ +# afl UM (random) + +Run afl over mutated code without UM prioritization. Randomly sample +list of generated mutants. + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/afl_um_random/fuzzer.py b/fuzzers/afl_um_random/fuzzer.py new file mode 100644 index 000000000..ccb0d3cab --- /dev/null +++ b/fuzzers/afl_um_random/fuzzer.py @@ -0,0 +1,196 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for afl fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.afl import fuzzer as afl_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + afl_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + os.environ['AFL_SKIP_CRASHES'] = "1" + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + afl_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + afl_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/afl_um_random/runner.Dockerfile b/fuzzers/afl_um_random/runner.Dockerfile new file mode 100644 index 000000000..0d6cf004e --- /dev/null +++ b/fuzzers/afl_um_random/runner.Dockerfile @@ -0,0 +1,15 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image diff --git a/fuzzers/aflplusplus_um_prioritize/description.md b/fuzzers/aflplusplus_um_prioritize/description.md index f7eb407ad..d5bfe6fea 100644 --- a/fuzzers/aflplusplus_um_prioritize/description.md +++ b/fuzzers/aflplusplus_um_prioritize/description.md @@ -1,13 +1,8 @@ -# aflplusplus +# aflplusplus UM (prioritize) -AFL++ fuzzer instance that has the following config active for all benchmarks: - - PCGUARD instrumentation - - cmplog feature - - dict2file feature - - "fast" power schedule - - persistent mode + shared memory test cases +Run aflplusplus over mutated code with UM prioritization -Repository: [https://github.com/AFLplusplus/AFLplusplus/](https://github.com/AFLplusplus/AFLplusplus/) +NOTE: This only works with C or C++ benchmarks. [builder.Dockerfile](builder.Dockerfile) [fuzzer.py](fuzzer.py) diff --git a/fuzzers/aflplusplus_um_prioritize/fuzzer.py b/fuzzers/aflplusplus_um_prioritize/fuzzer.py index 7b5d3831c..af16fd3a7 100755 --- a/fuzzers/aflplusplus_um_prioritize/fuzzer.py +++ b/fuzzers/aflplusplus_um_prioritize/fuzzer.py @@ -26,6 +26,7 @@ import random import shutil import filecmp +from subprocess import CalledProcessError import time import math import signal @@ -62,7 +63,7 @@ def signal_handler(signum, frame): signal.alarm(0) -def build(): # pylint: disable=too-many-locals,too-many-statements +def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-branches """Build benchmark.""" start_time = time.time() @@ -87,7 +88,7 @@ def build(): # pylint: disable=too-many-locals,too-many-statements total_fuzzing_time = int( os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) - source_extensions = [".c"] + source_extensions = [".c", ".cc", ".cpp"] num_mutants = math.ceil( (total_fuzzing_time * FUZZ_PROP) / DEFAULT_MUTANT_TIMEOUT) # Use heuristic to try to find benchmark directory, otherwise look for all @@ -173,8 +174,8 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.system(f"cp {source_file} {mutate_dir}/orig") os.system(f"cp {mutate_dir}/{mutant} {source_file}") try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ - .{num_non_buggy}" + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" os.system(f"rm -rf {out}/*") aflplusplus_fuzzer.build() @@ -192,6 +193,8 @@ def build(): # pylint: disable=too-many-locals,too-many-statements print(f"EQUAL {num_non_buggy}, ind: {ind}") except RuntimeError: pass + except CalledProcessError: + pass os.system(f"cp {mutate_dir}/orig {source_file}") ind += 1 except TimeoutException: diff --git a/fuzzers/aflplusplus_um_random/description.md b/fuzzers/aflplusplus_um_random/description.md index f7eb407ad..686a166cb 100644 --- a/fuzzers/aflplusplus_um_random/description.md +++ b/fuzzers/aflplusplus_um_random/description.md @@ -1,13 +1,9 @@ -# aflplusplus +# aflplusplus UM (random) -AFL++ fuzzer instance that has the following config active for all benchmarks: - - PCGUARD instrumentation - - cmplog feature - - dict2file feature - - "fast" power schedule - - persistent mode + shared memory test cases +Run aflplusplus over mutated code without UM prioritization. Randomly sample +list of generated mutants. -Repository: [https://github.com/AFLplusplus/AFLplusplus/](https://github.com/AFLplusplus/AFLplusplus/) +NOTE: This only works with C or C++ benchmarks. [builder.Dockerfile](builder.Dockerfile) [fuzzer.py](fuzzer.py) diff --git a/fuzzers/aflplusplus_um_random/fuzzer.py b/fuzzers/aflplusplus_um_random/fuzzer.py index 214452e5d..fda3d6f47 100644 --- a/fuzzers/aflplusplus_um_random/fuzzer.py +++ b/fuzzers/aflplusplus_um_random/fuzzer.py @@ -26,6 +26,7 @@ import random import shutil import filecmp +from subprocess import CalledProcessError import time import signal import math @@ -43,7 +44,6 @@ class TimeoutException(Exception): TOTAL_BUILD_TIME = 43200 # 12 hours FUZZ_PROP = 0.5 DEFAULT_MUTANT_TIMEOUT = 300 -PRIORITIZE_MULTIPLIER = 3 GRACE_TIME = 3600 # 1 hour in seconds @@ -85,7 +85,7 @@ def build(): # pylint: disable=too-many-locals,too-many-statements f"{mutate_bins}/{orig_fuzz_target}") benchmark = os.getenv("BENCHMARK") - source_extensions = [".c"] + source_extensions = [".c", ".cc", ".cpp"] # Use heuristic to try to find benchmark directory, # otherwise look for all files in the current directory. subdirs = [ @@ -144,8 +144,8 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.system(f"cp {mutate_dir}/{mutant} {source_file}") try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ - .{num_non_buggy}" + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" os.system(f"rm -rf {out}/*") aflplusplus_fuzzer.build() @@ -161,6 +161,8 @@ def build(): # pylint: disable=too-many-locals,too-many-statements print("EQUAL") except RuntimeError: pass + except CalledProcessError: + pass os.system(f"cp {mutate_dir}/orig {source_file}") ind += 1 except TimeoutException: diff --git a/fuzzers/eclipser_um_prioritize/builder.Dockerfile b/fuzzers/eclipser_um_prioritize/builder.Dockerfile new file mode 100644 index 000000000..b2a4ebf4f --- /dev/null +++ b/fuzzers/eclipser_um_prioritize/builder.Dockerfile @@ -0,0 +1,43 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Upgrade to avoid certs errors +RUN apt-get update && apt-get upgrade -y && \ + apt-get install -y apt-utils apt-transport-https ca-certificates + +# Download and compile AFL v2.56b, since Eclipser now adopts AFL as its random +# fuzzing module. Set AFL_NO_X86 to skip flaky tests. +RUN git clone https://github.com/google/AFL.git /afl && \ + cd /afl && \ + git checkout 82b5e359463238d790cadbe2dd494d6a4928bff3 && \ + AFL_NO_X86=1 make + +# Use afl_driver.cpp for AFL, and StandaloneFuzzTargetMain.c for Eclipser. +RUN apt-get update && \ + apt-get install wget -y && \ + wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \ + clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \ + clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \ + ar r /libAFL.a *.o && \ + wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c -O /StandaloneFuzzTargetMain.c && \ + clang -O2 -c /StandaloneFuzzTargetMain.c && \ + ar rc /libStandaloneFuzzTarget.a StandaloneFuzzTargetMain.o && \ + rm /StandaloneFuzzTargetMain.c diff --git a/fuzzers/eclipser_um_prioritize/description.md b/fuzzers/eclipser_um_prioritize/description.md new file mode 100644 index 000000000..9096a5663 --- /dev/null +++ b/fuzzers/eclipser_um_prioritize/description.md @@ -0,0 +1,9 @@ +# eclipser UM (prioritize) + +Run eclipser over mutated code with UM prioritization + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/eclipser_um_prioritize/fuzzer.py b/fuzzers/eclipser_um_prioritize/fuzzer.py new file mode 100755 index 000000000..6916f16e3 --- /dev/null +++ b/fuzzers/eclipser_um_prioritize/fuzzer.py @@ -0,0 +1,227 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for eclipser fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import math +import signal +from contextlib import contextmanager + +from fuzzers.eclipser import fuzzer as eclipser_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +PRIORITIZE_MULTIPLIER = 20 +GRACE_TIME = 3600 # 1 hour in seconds + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-branches + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + eclipser_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + + source_extensions = [".c", ".cc", ".cpp"] + num_mutants = math.ceil( + (total_fuzzing_time * FUZZ_PROP) / DEFAULT_MUTANT_TIMEOUT) + # Use heuristic to try to find benchmark directory, otherwise look for all + # files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + + num_prioritized = math.ceil( + (num_mutants * PRIORITIZE_MULTIPLIER) / len(source_files)) + + prioritize_map = {} + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants = [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf_8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + os.system(f"prioritize_mutants {mutate_dir}/mutants.txt \ + {mutate_dir}/prioritize_mutants_sorted.txt {num_prioritized}\ + --noSDPriority --sourceDir {src} --mutantDir {mutate_dir}") + prioritized_list = [] + with open(f"{mutate_dir}/prioritize_mutants_sorted.txt", + "r", + encoding="utf_8") as f_name: + prioritized_list = f_name.read().splitlines() + prioritize_map[source_file] = prioritized_list + + prioritized_keys = list(prioritize_map.keys()) + random.shuffle(prioritized_keys) + order = [] + ind = 0 + finished = False + + while not finished: + finished = True + for key in prioritized_keys: + if ind < len(prioritize_map[key]): + finished = False + order.append((key, ind)) + ind += 1 + print(order) + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print( + f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + eclipser_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + eclipser_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/eclipser_um_prioritize/runner.Dockerfile b/fuzzers/eclipser_um_prioritize/runner.Dockerfile new file mode 100644 index 000000000..9d620dcca --- /dev/null +++ b/fuzzers/eclipser_um_prioritize/runner.Dockerfile @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# Install dotnet, qemu and other Eclipser deps. +RUN sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list +RUN apt-get update -y && \ + apt-get build-dep -y qemu && \ + apt-get install -y \ + apt-transport-https \ + libtool \ + libtool-bin \ + wget \ + automake \ + autoconf \ + bison \ + git \ + gdb + +# Use a copy of +# https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb +# to avoid network flakiness. +RUN wget -q https://storage.googleapis.com/fuzzbench-files/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \ + dpkg -i packages-microsoft-prod.deb && \ + apt-get update -y && \ + apt-get install -y dotnet-sdk-2.1 dotnet-runtime-2.1 && \ + rm packages-microsoft-prod.deb + +# Build Eclipser. +RUN git clone https://github.com/SoftSec-KAIST/Eclipser.git /Eclipser && \ + cd /Eclipser && \ + git checkout 310220649a4d790f8bc858ef85873399bba79a8c && \ + make + +ENV AFL_MAP_SIZE=2222222 +ENV PATH="$PATH:/out" +ENV AFL_SKIP_CPUFREQ=1 +ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 +ENV AFL_TESTCACHE_SIZE=2 diff --git a/fuzzers/eclipser_um_random/builder.Dockerfile b/fuzzers/eclipser_um_random/builder.Dockerfile new file mode 100644 index 000000000..b2a4ebf4f --- /dev/null +++ b/fuzzers/eclipser_um_random/builder.Dockerfile @@ -0,0 +1,43 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Upgrade to avoid certs errors +RUN apt-get update && apt-get upgrade -y && \ + apt-get install -y apt-utils apt-transport-https ca-certificates + +# Download and compile AFL v2.56b, since Eclipser now adopts AFL as its random +# fuzzing module. Set AFL_NO_X86 to skip flaky tests. +RUN git clone https://github.com/google/AFL.git /afl && \ + cd /afl && \ + git checkout 82b5e359463238d790cadbe2dd494d6a4928bff3 && \ + AFL_NO_X86=1 make + +# Use afl_driver.cpp for AFL, and StandaloneFuzzTargetMain.c for Eclipser. +RUN apt-get update && \ + apt-get install wget -y && \ + wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \ + clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \ + clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \ + ar r /libAFL.a *.o && \ + wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c -O /StandaloneFuzzTargetMain.c && \ + clang -O2 -c /StandaloneFuzzTargetMain.c && \ + ar rc /libStandaloneFuzzTarget.a StandaloneFuzzTargetMain.o && \ + rm /StandaloneFuzzTargetMain.c diff --git a/fuzzers/eclipser_um_random/description.md b/fuzzers/eclipser_um_random/description.md new file mode 100644 index 000000000..686a166cb --- /dev/null +++ b/fuzzers/eclipser_um_random/description.md @@ -0,0 +1,10 @@ +# aflplusplus UM (random) + +Run aflplusplus over mutated code without UM prioritization. Randomly sample +list of generated mutants. + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/eclipser_um_random/fuzzer.py b/fuzzers/eclipser_um_random/fuzzer.py new file mode 100644 index 000000000..a6f5d7ed7 --- /dev/null +++ b/fuzzers/eclipser_um_random/fuzzer.py @@ -0,0 +1,196 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for eclipser fuzzer.""" + + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.eclipser import fuzzer as eclipser_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + eclipser_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + eclipser_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + eclipser_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/eclipser_um_random/runner.Dockerfile b/fuzzers/eclipser_um_random/runner.Dockerfile new file mode 100644 index 000000000..9d620dcca --- /dev/null +++ b/fuzzers/eclipser_um_random/runner.Dockerfile @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# Install dotnet, qemu and other Eclipser deps. +RUN sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list +RUN apt-get update -y && \ + apt-get build-dep -y qemu && \ + apt-get install -y \ + apt-transport-https \ + libtool \ + libtool-bin \ + wget \ + automake \ + autoconf \ + bison \ + git \ + gdb + +# Use a copy of +# https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb +# to avoid network flakiness. +RUN wget -q https://storage.googleapis.com/fuzzbench-files/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \ + dpkg -i packages-microsoft-prod.deb && \ + apt-get update -y && \ + apt-get install -y dotnet-sdk-2.1 dotnet-runtime-2.1 && \ + rm packages-microsoft-prod.deb + +# Build Eclipser. +RUN git clone https://github.com/SoftSec-KAIST/Eclipser.git /Eclipser && \ + cd /Eclipser && \ + git checkout 310220649a4d790f8bc858ef85873399bba79a8c && \ + make + +ENV AFL_MAP_SIZE=2222222 +ENV PATH="$PATH:/out" +ENV AFL_SKIP_CPUFREQ=1 +ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 +ENV AFL_TESTCACHE_SIZE=2 diff --git a/fuzzers/honggfuzz_um_prioritize/builder.Dockerfile b/fuzzers/honggfuzz_um_prioritize/builder.Dockerfile new file mode 100644 index 000000000..d5c2c5dca --- /dev/null +++ b/fuzzers/honggfuzz_um_prioritize/builder.Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# honggfuzz requires libfd and libunwid. +RUN apt-get update -y && \ + apt-get install -y \ + libbfd-dev \ + libunwind-dev \ + libblocksruntime-dev \ + liblzma-dev + +# Download honggfuz version 2.3.1 + 0b4cd5b1c4cf26b7e022dc1deb931d9318c054cb +# Set CFLAGS use honggfuzz's defaults except for -mnative which can build CPU +# dependent code that may not work on the machines we actually fuzz on. +# Create an empty object file which will become the FUZZER_LIB lib (since +# honggfuzz doesn't need this when hfuzz-clang(++) is used). +RUN git clone https://github.com/google/honggfuzz.git /honggfuzz && \ + cd /honggfuzz && \ + git checkout 0b4cd5b1c4cf26b7e022dc1deb931d9318c054cb && \ + CFLAGS="-O3 -funroll-loops" make && \ + touch empty_lib.c && \ + cc -c -o empty_lib.o empty_lib.c diff --git a/fuzzers/honggfuzz_um_prioritize/description.md b/fuzzers/honggfuzz_um_prioritize/description.md new file mode 100644 index 000000000..ca04efdba --- /dev/null +++ b/fuzzers/honggfuzz_um_prioritize/description.md @@ -0,0 +1,9 @@ +# honggfuzz UM (prioritize) + +Run honggfuzz over mutated code with UM prioritization + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/honggfuzz_um_prioritize/fuzzer.py b/fuzzers/honggfuzz_um_prioritize/fuzzer.py new file mode 100755 index 000000000..421675812 --- /dev/null +++ b/fuzzers/honggfuzz_um_prioritize/fuzzer.py @@ -0,0 +1,227 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for honggfuzz fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import math +import signal +from contextlib import contextmanager + +from fuzzers.honggfuzz import fuzzer as honggfuzz_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +PRIORITIZE_MULTIPLIER = 20 +GRACE_TIME = 3600 # 1 hour in seconds + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-branches + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + honggfuzz_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + + source_extensions = [".c", ".cc", ".cpp"] + num_mutants = math.ceil( + (total_fuzzing_time * FUZZ_PROP) / DEFAULT_MUTANT_TIMEOUT) + # Use heuristic to try to find benchmark directory, otherwise look for all + # files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + + num_prioritized = math.ceil( + (num_mutants * PRIORITIZE_MULTIPLIER) / len(source_files)) + + prioritize_map = {} + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants = [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf_8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + os.system(f"prioritize_mutants {mutate_dir}/mutants.txt \ + {mutate_dir}/prioritize_mutants_sorted.txt {num_prioritized}\ + --noSDPriority --sourceDir {src} --mutantDir {mutate_dir}") + prioritized_list = [] + with open(f"{mutate_dir}/prioritize_mutants_sorted.txt", + "r", + encoding="utf_8") as f_name: + prioritized_list = f_name.read().splitlines() + prioritize_map[source_file] = prioritized_list + + prioritized_keys = list(prioritize_map.keys()) + random.shuffle(prioritized_keys) + order = [] + ind = 0 + finished = False + + while not finished: + finished = True + for key in prioritized_keys: + if ind < len(prioritize_map[key]): + finished = False + order.append((key, ind)) + ind += 1 + print(order) + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print( + f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/honggfuzz_um_prioritize/runner.Dockerfile b/fuzzers/honggfuzz_um_prioritize/runner.Dockerfile new file mode 100644 index 000000000..f3eb30039 --- /dev/null +++ b/fuzzers/honggfuzz_um_prioritize/runner.Dockerfile @@ -0,0 +1,18 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# honggfuzz requires libfd and libunwid +RUN apt-get update -y && apt-get install -y libbfd-dev libunwind-dev diff --git a/fuzzers/honggfuzz_um_random/builder.Dockerfile b/fuzzers/honggfuzz_um_random/builder.Dockerfile new file mode 100644 index 000000000..d5c2c5dca --- /dev/null +++ b/fuzzers/honggfuzz_um_random/builder.Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# honggfuzz requires libfd and libunwid. +RUN apt-get update -y && \ + apt-get install -y \ + libbfd-dev \ + libunwind-dev \ + libblocksruntime-dev \ + liblzma-dev + +# Download honggfuz version 2.3.1 + 0b4cd5b1c4cf26b7e022dc1deb931d9318c054cb +# Set CFLAGS use honggfuzz's defaults except for -mnative which can build CPU +# dependent code that may not work on the machines we actually fuzz on. +# Create an empty object file which will become the FUZZER_LIB lib (since +# honggfuzz doesn't need this when hfuzz-clang(++) is used). +RUN git clone https://github.com/google/honggfuzz.git /honggfuzz && \ + cd /honggfuzz && \ + git checkout 0b4cd5b1c4cf26b7e022dc1deb931d9318c054cb && \ + CFLAGS="-O3 -funroll-loops" make && \ + touch empty_lib.c && \ + cc -c -o empty_lib.o empty_lib.c diff --git a/fuzzers/honggfuzz_um_random/description.md b/fuzzers/honggfuzz_um_random/description.md new file mode 100644 index 000000000..686a166cb --- /dev/null +++ b/fuzzers/honggfuzz_um_random/description.md @@ -0,0 +1,10 @@ +# aflplusplus UM (random) + +Run aflplusplus over mutated code without UM prioritization. Randomly sample +list of generated mutants. + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/honggfuzz_um_random/fuzzer.py b/fuzzers/honggfuzz_um_random/fuzzer.py new file mode 100644 index 000000000..a223f58b5 --- /dev/null +++ b/fuzzers/honggfuzz_um_random/fuzzer.py @@ -0,0 +1,196 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for honggfuzz fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.honggfuzz import fuzzer as honggfuzz_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + honggfuzz_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + os.environ['AFL_SKIP_CRASHES'] = "1" + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/honggfuzz_um_random/runner.Dockerfile b/fuzzers/honggfuzz_um_random/runner.Dockerfile new file mode 100644 index 000000000..f3eb30039 --- /dev/null +++ b/fuzzers/honggfuzz_um_random/runner.Dockerfile @@ -0,0 +1,18 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# honggfuzz requires libfd and libunwid +RUN apt-get update -y && apt-get install -y libbfd-dev libunwind-dev diff --git a/fuzzers/libfuzzer_um_prioritize/builder.Dockerfile b/fuzzers/libfuzzer_um_prioritize/builder.Dockerfile new file mode 100644 index 000000000..c70af8e58 --- /dev/null +++ b/fuzzers/libfuzzer_um_prioritize/builder.Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +RUN git clone https://github.com/llvm/llvm-project.git /llvm-project && \ + cd /llvm-project/ && \ + git checkout 5cda4dc7b4d28fcd11307d4234c513ff779a1c6f && \ + cd compiler-rt/lib/fuzzer && \ + (for f in *.cpp; do \ + clang++ -stdlib=libc++ -fPIC -O2 -std=c++11 $f -c & \ + done && wait) && \ + ar r /usr/lib/libFuzzer.a *.o \ No newline at end of file diff --git a/fuzzers/libfuzzer_um_prioritize/description.md b/fuzzers/libfuzzer_um_prioritize/description.md new file mode 100644 index 000000000..7dc8a3db5 --- /dev/null +++ b/fuzzers/libfuzzer_um_prioritize/description.md @@ -0,0 +1,9 @@ +# libfuzzer UM (prioritize) + +Run libfuzzer over mutated code with UM prioritization + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/libfuzzer_um_prioritize/fuzzer.py b/fuzzers/libfuzzer_um_prioritize/fuzzer.py new file mode 100755 index 000000000..512e90936 --- /dev/null +++ b/fuzzers/libfuzzer_um_prioritize/fuzzer.py @@ -0,0 +1,224 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for libfuzzer fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +import time +import math +import signal +from contextlib import contextmanager + +from fuzzers.libfuzzer import fuzzer as libfuzzer_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +PRIORITIZE_MULTIPLIER = 20 +GRACE_TIME = 3600 # 1 hour in seconds + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-branches + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + libfuzzer_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + + source_extensions = [".c", ".cc", ".cpp"] + num_mutants = math.ceil( + (total_fuzzing_time * FUZZ_PROP) / DEFAULT_MUTANT_TIMEOUT) + # Use heuristic to try to find benchmark directory, otherwise look for all + # files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + + num_prioritized = math.ceil( + (num_mutants * PRIORITIZE_MULTIPLIER) / len(source_files)) + + prioritize_map = {} + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants = [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf_8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + os.system(f"prioritize_mutants {mutate_dir}/mutants.txt \ + {mutate_dir}/prioritize_mutants_sorted.txt {num_prioritized}\ + --noSDPriority --sourceDir {src} --mutantDir {mutate_dir}") + prioritized_list = [] + with open(f"{mutate_dir}/prioritize_mutants_sorted.txt", + "r", + encoding="utf_8") as f_name: + prioritized_list = f_name.read().splitlines() + prioritize_map[source_file] = prioritized_list + + prioritized_keys = list(prioritize_map.keys()) + random.shuffle(prioritized_keys) + order = [] + ind = 0 + finished = False + + while not finished: + finished = True + for key in prioritized_keys: + if ind < len(prioritize_map[key]): + finished = False + order.append((key, ind)) + ind += 1 + print(order) + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + libfuzzer_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print( + f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + libfuzzer_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + libfuzzer_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + libfuzzer_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/libfuzzer_um_prioritize/runner.Dockerfile b/fuzzers/libfuzzer_um_prioritize/runner.Dockerfile new file mode 100644 index 000000000..0d6cf004e --- /dev/null +++ b/fuzzers/libfuzzer_um_prioritize/runner.Dockerfile @@ -0,0 +1,15 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image diff --git a/fuzzers/libfuzzer_um_random/builder.Dockerfile b/fuzzers/libfuzzer_um_random/builder.Dockerfile new file mode 100644 index 000000000..c70af8e58 --- /dev/null +++ b/fuzzers/libfuzzer_um_random/builder.Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +RUN git clone https://github.com/llvm/llvm-project.git /llvm-project && \ + cd /llvm-project/ && \ + git checkout 5cda4dc7b4d28fcd11307d4234c513ff779a1c6f && \ + cd compiler-rt/lib/fuzzer && \ + (for f in *.cpp; do \ + clang++ -stdlib=libc++ -fPIC -O2 -std=c++11 $f -c & \ + done && wait) && \ + ar r /usr/lib/libFuzzer.a *.o \ No newline at end of file diff --git a/fuzzers/libfuzzer_um_random/description.md b/fuzzers/libfuzzer_um_random/description.md new file mode 100644 index 000000000..7dc8a3db5 --- /dev/null +++ b/fuzzers/libfuzzer_um_random/description.md @@ -0,0 +1,9 @@ +# libfuzzer UM (prioritize) + +Run libfuzzer over mutated code with UM prioritization + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/libfuzzer_um_random/fuzzer.py b/fuzzers/libfuzzer_um_random/fuzzer.py new file mode 100755 index 000000000..bc3b42d02 --- /dev/null +++ b/fuzzers/libfuzzer_um_random/fuzzer.py @@ -0,0 +1,192 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for libfuzzer fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.libfuzzer import fuzzer as libfuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + libfuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + libfuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + libfuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + libfuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + libfuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/libfuzzer_um_random/runner.Dockerfile b/fuzzers/libfuzzer_um_random/runner.Dockerfile new file mode 100644 index 000000000..0d6cf004e --- /dev/null +++ b/fuzzers/libfuzzer_um_random/runner.Dockerfile @@ -0,0 +1,15 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image diff --git a/service/experiment-requests.yaml b/service/experiment-requests.yaml index 23cdd909d..e0c8cbb1f 100644 --- a/service/experiment-requests.yaml +++ b/service/experiment-requests.yaml @@ -19,6 +19,12 @@ # You can run "make presubmit" to do basic validation on this file. # Please add new experiment requests towards the top of this file. # +- experiment: 2022-08-31-aflpp-um + description: "afl++ um coverage experiments (compare our fuzzers against afl++)" + fuzzers: + - aflplusplus_um_prioritize + - aflplusplus_um_random + - aflplusplus - experiment: 2022-08-25-aflpp-um description: "afl++ um coverage experiments (compare our fuzzers against afl++)" From 1e1513e7a623d10a55752af8cd15643659ffbc6f Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Mon, 26 Sep 2022 15:28:37 -0400 Subject: [PATCH 04/18] Add UM parallel modes --- fuzzers/afl_um_parallel/builder.Dockerfile | 36 +++ fuzzers/afl_um_parallel/description.md | 9 + fuzzers/afl_um_parallel/fuzzer.py | 200 +++++++++++++++++ fuzzers/afl_um_parallel/runner.Dockerfile | 15 ++ .../builder.Dockerfile | 40 ++++ .../aflplusplus_um_parallel/description.md | 9 + fuzzers/aflplusplus_um_parallel/fuzzer.py | 207 ++++++++++++++++++ .../aflplusplus_um_parallel/runner.Dockerfile | 23 ++ .../eclipser_um_parallel/builder.Dockerfile | 43 ++++ fuzzers/eclipser_um_parallel/description.md | 9 + fuzzers/eclipser_um_parallel/fuzzer.py | 199 +++++++++++++++++ .../eclipser_um_parallel/runner.Dockerfile | 51 +++++ .../honggfuzz_um_parallel/builder.Dockerfile | 40 ++++ fuzzers/honggfuzz_um_parallel/description.md | 9 + fuzzers/honggfuzz_um_parallel/fuzzer.py | 200 +++++++++++++++++ .../honggfuzz_um_parallel/runner.Dockerfile | 18 ++ .../libfuzzer_um_parallel/builder.Dockerfile | 29 +++ fuzzers/libfuzzer_um_parallel/description.md | 9 + fuzzers/libfuzzer_um_parallel/fuzzer.py | 199 +++++++++++++++++ .../libfuzzer_um_parallel/runner.Dockerfile | 15 ++ 20 files changed, 1360 insertions(+) create mode 100644 fuzzers/afl_um_parallel/builder.Dockerfile create mode 100644 fuzzers/afl_um_parallel/description.md create mode 100644 fuzzers/afl_um_parallel/fuzzer.py create mode 100644 fuzzers/afl_um_parallel/runner.Dockerfile create mode 100644 fuzzers/aflplusplus_um_parallel/builder.Dockerfile create mode 100644 fuzzers/aflplusplus_um_parallel/description.md create mode 100644 fuzzers/aflplusplus_um_parallel/fuzzer.py create mode 100644 fuzzers/aflplusplus_um_parallel/runner.Dockerfile create mode 100644 fuzzers/eclipser_um_parallel/builder.Dockerfile create mode 100644 fuzzers/eclipser_um_parallel/description.md create mode 100644 fuzzers/eclipser_um_parallel/fuzzer.py create mode 100644 fuzzers/eclipser_um_parallel/runner.Dockerfile create mode 100644 fuzzers/honggfuzz_um_parallel/builder.Dockerfile create mode 100644 fuzzers/honggfuzz_um_parallel/description.md create mode 100644 fuzzers/honggfuzz_um_parallel/fuzzer.py create mode 100644 fuzzers/honggfuzz_um_parallel/runner.Dockerfile create mode 100644 fuzzers/libfuzzer_um_parallel/builder.Dockerfile create mode 100644 fuzzers/libfuzzer_um_parallel/description.md create mode 100755 fuzzers/libfuzzer_um_parallel/fuzzer.py create mode 100644 fuzzers/libfuzzer_um_parallel/runner.Dockerfile diff --git a/fuzzers/afl_um_parallel/builder.Dockerfile b/fuzzers/afl_um_parallel/builder.Dockerfile new file mode 100644 index 000000000..253ac9f87 --- /dev/null +++ b/fuzzers/afl_um_parallel/builder.Dockerfile @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Download and compile AFL v2.57b. +# Set AFL_NO_X86 to skip flaky tests. +RUN git clone https://github.com/google/AFL.git /afl && \ + cd /afl && \ + git checkout 61037103ae3722c8060ff7082994836a794f978e && \ + AFL_NO_X86=1 make + +# Use afl_driver.cpp from LLVM as our fuzzing library. +RUN apt-get update && \ + apt-get install wget -y && \ + wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \ + clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \ + clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \ + ar r /libAFL.a *.o + diff --git a/fuzzers/afl_um_parallel/description.md b/fuzzers/afl_um_parallel/description.md new file mode 100644 index 000000000..3086a07c0 --- /dev/null +++ b/fuzzers/afl_um_parallel/description.md @@ -0,0 +1,9 @@ +# afl UM (parallel) + +Run afl over mutated code in parallel. + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/afl_um_parallel/fuzzer.py b/fuzzers/afl_um_parallel/fuzzer.py new file mode 100644 index 000000000..2541906e5 --- /dev/null +++ b/fuzzers/afl_um_parallel/fuzzer.py @@ -0,0 +1,200 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for afl fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.afl import fuzzer as afl_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds +MAX_MUTANTS = 200000 + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + afl_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + random.shuffle(source_files) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + if len(mutants) > MAX_MUTANTS: + break + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + os.environ['AFL_SKIP_CRASHES'] = "1" + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + afl_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + afl_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/afl_um_parallel/runner.Dockerfile b/fuzzers/afl_um_parallel/runner.Dockerfile new file mode 100644 index 000000000..0d6cf004e --- /dev/null +++ b/fuzzers/afl_um_parallel/runner.Dockerfile @@ -0,0 +1,15 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image diff --git a/fuzzers/aflplusplus_um_parallel/builder.Dockerfile b/fuzzers/aflplusplus_um_parallel/builder.Dockerfile new file mode 100644 index 000000000..abd77021b --- /dev/null +++ b/fuzzers/aflplusplus_um_parallel/builder.Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Install libstdc++ to use llvm_mode. +RUN apt-get update && \ + apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \ + libglib2.0-dev libpixman-1-dev python3-setuptools unzip \ + apt-utils apt-transport-https ca-certificates + +# Download and compile afl++. +RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \ + cd /afl && \ + git checkout b847e0f414e7b310e1a68bc501d4e2453bfce70e + +# Build without Python support as we don't need it. +# Set AFL_NO_X86 to skip flaky tests. +RUN cd /afl && unset CFLAGS && unset CXXFLAGS && \ + export CC=clang && export AFL_NO_X86=1 && \ + PYTHON_INCLUDE=/ make && make install && \ + make -C utils/aflpp_driver && \ + cp utils/aflpp_driver/libAFLDriver.a / + diff --git a/fuzzers/aflplusplus_um_parallel/description.md b/fuzzers/aflplusplus_um_parallel/description.md new file mode 100644 index 000000000..2ff91d2fd --- /dev/null +++ b/fuzzers/aflplusplus_um_parallel/description.md @@ -0,0 +1,9 @@ +# aflplusplus UM (parallel) + +Run aflplusplus over mutated code with parallel. + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/aflplusplus_um_parallel/fuzzer.py b/fuzzers/aflplusplus_um_parallel/fuzzer.py new file mode 100644 index 000000000..91bcf8c40 --- /dev/null +++ b/fuzzers/aflplusplus_um_parallel/fuzzer.py @@ -0,0 +1,207 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for AFLplusplus fuzzer.""" + +# This optimized afl++ variant should always be run together with +# "aflplusplus" to show the difference - a default configured afl++ vs. +# a hand-crafted optimized one. afl++ is configured not to enable the good +# stuff by default to be as close to vanilla afl as possible. +# But this means that the good stuff is hidden away in this benchmark +# otherwise. + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.aflplusplus import fuzzer as aflplusplus_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds +MAX_MUTANTS = 200000 + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + aflplusplus_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + random.shuffle(source_files) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + if len(mutants) > MAX_MUTANTS: + break + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + os.environ['AFL_SKIP_CRASHES'] = "1" + + for mutant in mutants[:num_mutants]: + try: + with time_limit(timeout): + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/aflplusplus_um_parallel/runner.Dockerfile b/fuzzers/aflplusplus_um_parallel/runner.Dockerfile new file mode 100644 index 000000000..7aa1da8e4 --- /dev/null +++ b/fuzzers/aflplusplus_um_parallel/runner.Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# This makes interactive docker runs painless: +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out" +#ENV AFL_MAP_SIZE=2621440 +ENV PATH="$PATH:/out" +ENV AFL_SKIP_CPUFREQ=1 +ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 +ENV AFL_TESTCACHE_SIZE=2 diff --git a/fuzzers/eclipser_um_parallel/builder.Dockerfile b/fuzzers/eclipser_um_parallel/builder.Dockerfile new file mode 100644 index 000000000..b2a4ebf4f --- /dev/null +++ b/fuzzers/eclipser_um_parallel/builder.Dockerfile @@ -0,0 +1,43 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Upgrade to avoid certs errors +RUN apt-get update && apt-get upgrade -y && \ + apt-get install -y apt-utils apt-transport-https ca-certificates + +# Download and compile AFL v2.56b, since Eclipser now adopts AFL as its random +# fuzzing module. Set AFL_NO_X86 to skip flaky tests. +RUN git clone https://github.com/google/AFL.git /afl && \ + cd /afl && \ + git checkout 82b5e359463238d790cadbe2dd494d6a4928bff3 && \ + AFL_NO_X86=1 make + +# Use afl_driver.cpp for AFL, and StandaloneFuzzTargetMain.c for Eclipser. +RUN apt-get update && \ + apt-get install wget -y && \ + wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \ + clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \ + clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \ + ar r /libAFL.a *.o && \ + wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c -O /StandaloneFuzzTargetMain.c && \ + clang -O2 -c /StandaloneFuzzTargetMain.c && \ + ar rc /libStandaloneFuzzTarget.a StandaloneFuzzTargetMain.o && \ + rm /StandaloneFuzzTargetMain.c diff --git a/fuzzers/eclipser_um_parallel/description.md b/fuzzers/eclipser_um_parallel/description.md new file mode 100644 index 000000000..9163c5cb6 --- /dev/null +++ b/fuzzers/eclipser_um_parallel/description.md @@ -0,0 +1,9 @@ +# aflplusplus UM (parallel) + +Run aflplusplus over mutated code in parallel. + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/eclipser_um_parallel/fuzzer.py b/fuzzers/eclipser_um_parallel/fuzzer.py new file mode 100644 index 000000000..ef09a361b --- /dev/null +++ b/fuzzers/eclipser_um_parallel/fuzzer.py @@ -0,0 +1,199 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for eclipser fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.eclipser import fuzzer as eclipser_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds +MAX_MUTANTS = 200000 + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + eclipser_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + random.shuffle(source_files) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + if len(mutants) > MAX_MUTANTS: + break + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + eclipser_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + eclipser_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/eclipser_um_parallel/runner.Dockerfile b/fuzzers/eclipser_um_parallel/runner.Dockerfile new file mode 100644 index 000000000..9d620dcca --- /dev/null +++ b/fuzzers/eclipser_um_parallel/runner.Dockerfile @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# Install dotnet, qemu and other Eclipser deps. +RUN sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list +RUN apt-get update -y && \ + apt-get build-dep -y qemu && \ + apt-get install -y \ + apt-transport-https \ + libtool \ + libtool-bin \ + wget \ + automake \ + autoconf \ + bison \ + git \ + gdb + +# Use a copy of +# https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb +# to avoid network flakiness. +RUN wget -q https://storage.googleapis.com/fuzzbench-files/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \ + dpkg -i packages-microsoft-prod.deb && \ + apt-get update -y && \ + apt-get install -y dotnet-sdk-2.1 dotnet-runtime-2.1 && \ + rm packages-microsoft-prod.deb + +# Build Eclipser. +RUN git clone https://github.com/SoftSec-KAIST/Eclipser.git /Eclipser && \ + cd /Eclipser && \ + git checkout 310220649a4d790f8bc858ef85873399bba79a8c && \ + make + +ENV AFL_MAP_SIZE=2222222 +ENV PATH="$PATH:/out" +ENV AFL_SKIP_CPUFREQ=1 +ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 +ENV AFL_TESTCACHE_SIZE=2 diff --git a/fuzzers/honggfuzz_um_parallel/builder.Dockerfile b/fuzzers/honggfuzz_um_parallel/builder.Dockerfile new file mode 100644 index 000000000..d5c2c5dca --- /dev/null +++ b/fuzzers/honggfuzz_um_parallel/builder.Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# honggfuzz requires libfd and libunwid. +RUN apt-get update -y && \ + apt-get install -y \ + libbfd-dev \ + libunwind-dev \ + libblocksruntime-dev \ + liblzma-dev + +# Download honggfuz version 2.3.1 + 0b4cd5b1c4cf26b7e022dc1deb931d9318c054cb +# Set CFLAGS use honggfuzz's defaults except for -mnative which can build CPU +# dependent code that may not work on the machines we actually fuzz on. +# Create an empty object file which will become the FUZZER_LIB lib (since +# honggfuzz doesn't need this when hfuzz-clang(++) is used). +RUN git clone https://github.com/google/honggfuzz.git /honggfuzz && \ + cd /honggfuzz && \ + git checkout 0b4cd5b1c4cf26b7e022dc1deb931d9318c054cb && \ + CFLAGS="-O3 -funroll-loops" make && \ + touch empty_lib.c && \ + cc -c -o empty_lib.o empty_lib.c diff --git a/fuzzers/honggfuzz_um_parallel/description.md b/fuzzers/honggfuzz_um_parallel/description.md new file mode 100644 index 000000000..9163c5cb6 --- /dev/null +++ b/fuzzers/honggfuzz_um_parallel/description.md @@ -0,0 +1,9 @@ +# aflplusplus UM (parallel) + +Run aflplusplus over mutated code in parallel. + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/honggfuzz_um_parallel/fuzzer.py b/fuzzers/honggfuzz_um_parallel/fuzzer.py new file mode 100644 index 000000000..38e9ccb43 --- /dev/null +++ b/fuzzers/honggfuzz_um_parallel/fuzzer.py @@ -0,0 +1,200 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for honggfuzz fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.honggfuzz import fuzzer as honggfuzz_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds +MAX_MUTANTS = 200000 + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + honggfuzz_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + random.shuffle(source_files) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + if len(mutants) > MAX_MUTANTS: + break + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + os.environ['AFL_SKIP_CRASHES'] = "1" + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/honggfuzz_um_parallel/runner.Dockerfile b/fuzzers/honggfuzz_um_parallel/runner.Dockerfile new file mode 100644 index 000000000..f3eb30039 --- /dev/null +++ b/fuzzers/honggfuzz_um_parallel/runner.Dockerfile @@ -0,0 +1,18 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# honggfuzz requires libfd and libunwid +RUN apt-get update -y && apt-get install -y libbfd-dev libunwind-dev diff --git a/fuzzers/libfuzzer_um_parallel/builder.Dockerfile b/fuzzers/libfuzzer_um_parallel/builder.Dockerfile new file mode 100644 index 000000000..c70af8e58 --- /dev/null +++ b/fuzzers/libfuzzer_um_parallel/builder.Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +RUN git clone https://github.com/llvm/llvm-project.git /llvm-project && \ + cd /llvm-project/ && \ + git checkout 5cda4dc7b4d28fcd11307d4234c513ff779a1c6f && \ + cd compiler-rt/lib/fuzzer && \ + (for f in *.cpp; do \ + clang++ -stdlib=libc++ -fPIC -O2 -std=c++11 $f -c & \ + done && wait) && \ + ar r /usr/lib/libFuzzer.a *.o \ No newline at end of file diff --git a/fuzzers/libfuzzer_um_parallel/description.md b/fuzzers/libfuzzer_um_parallel/description.md new file mode 100644 index 000000000..41997bf7c --- /dev/null +++ b/fuzzers/libfuzzer_um_parallel/description.md @@ -0,0 +1,9 @@ +# libfuzzer UM (parallel) + +Run libfuzzer over mutated code in parallel + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/libfuzzer_um_parallel/fuzzer.py b/fuzzers/libfuzzer_um_parallel/fuzzer.py new file mode 100755 index 000000000..d0959f3e5 --- /dev/null +++ b/fuzzers/libfuzzer_um_parallel/fuzzer.py @@ -0,0 +1,199 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for libfuzzer fuzzer.""" + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.libfuzzer import fuzzer as libfuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds +MAX_MUTANTS = 200000 + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + libfuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + random.shuffle(source_files) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + if len(mutants) > MAX_MUTANTS: + break + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + libfuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + libfuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.mkdir("/storage") + os.mkdir(input_corpus_dir) + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + try: + with time_limit(timeout): + libfuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + libfuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/libfuzzer_um_parallel/runner.Dockerfile b/fuzzers/libfuzzer_um_parallel/runner.Dockerfile new file mode 100644 index 000000000..0d6cf004e --- /dev/null +++ b/fuzzers/libfuzzer_um_parallel/runner.Dockerfile @@ -0,0 +1,15 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image From 0afae166d50397bc1c84668b7b9d1c9dfc392510 Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Mon, 26 Sep 2022 16:46:34 -0400 Subject: [PATCH 05/18] Add new fuzzers --- .github/workflows/fuzzers.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml index b70c2bd49..6f5734bc2 100644 --- a/.github/workflows/fuzzers.yml +++ b/.github/workflows/fuzzers.yml @@ -21,11 +21,13 @@ jobs: - aflfast - afl_um_prioritize - afl_um_random + - afl_um_parallel - aflplusplus - aflplusplus_optimal - aflplusplus_tracepc - aflplusplus_um_prioritize - aflplusplus_um_random + - aflplusplus_um_parallel - aflsmart - centipede - entropic @@ -33,11 +35,13 @@ jobs: - honggfuzz - honggfuzz_um_random - honggfuzz_um_prioritize + - honggfuzz_um_parallel - lafintel - klee - libfuzzer - libfuzzer_um_random - libfuzzer_um_prioritize + - libfuzzer_um_parallel - libfuzzer_dataflow - libfuzzer_dataflow_load - libfuzzer_dataflow_store @@ -53,6 +57,7 @@ jobs: - eclipser - eclipser_um_prioritize - eclipser_um_random + - eclipser_um_parallel - afl_qemu - honggfuzz_qemu - weizz_qemu From f01557eb3e233a0112cf543279f60ccbff812162 Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Mon, 26 Sep 2022 17:46:22 -0400 Subject: [PATCH 06/18] Adding comparison experiment --- service/experiment-requests.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/service/experiment-requests.yaml b/service/experiment-requests.yaml index 898ff5254..198ec2c18 100644 --- a/service/experiment-requests.yaml +++ b/service/experiment-requests.yaml @@ -19,6 +19,19 @@ # You can run "make presubmit" to do basic validation on this file. # Please add new experiment requests towards the top of this file. # + +- experiment: 2022-09-26-aflpp-um + description: "afl++ and libfuzzer um coverage experiments (compare our fuzzers against afl++)" + fuzzers: + - aflplusplus_um_prioritize + - aflplusplus_um_random + - aflplusplus_um_parallel + - aflplusplus + - libfuzzer_um_prioritize + - libfuzzer_um_random + - libfuzzer_um_parallel + - libfuzzer + - experiment: 2022-09-22-aflpp-um description: "afl++ um coverage experiments (compare our fuzzers against afl++)" fuzzers: From e19f05c286618371ebbf30d53abdc77dd5bf2678 Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Tue, 4 Oct 2022 12:17:50 -0400 Subject: [PATCH 07/18] Fixing random fuzzer for aflplusplus, should hopefully resolve build issues --- fuzzers/aflplusplus_um_random/fuzzer.py | 100 ++++++++++++------------ 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/fuzzers/aflplusplus_um_random/fuzzer.py b/fuzzers/aflplusplus_um_random/fuzzer.py index 7355d0767..1a539989d 100644 --- a/fuzzers/aflplusplus_um_random/fuzzer.py +++ b/fuzzers/aflplusplus_um_random/fuzzer.py @@ -131,47 +131,47 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src), utils.restore_directory(work): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") aflplusplus_fuzzer.build() @@ -191,18 +191,20 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) os.environ['AFL_SKIP_CRASHES'] = "1" for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory(output_corpus): + try: + with time_limit(timeout): + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) From 65f7c6017947b7c89e50d6ecf41cbc3d287fd1e5 Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Tue, 4 Oct 2022 12:19:02 -0400 Subject: [PATCH 08/18] Fixing formatting issues --- fuzzers/aflplusplus_um_random/fuzzer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fuzzers/aflplusplus_um_random/fuzzer.py b/fuzzers/aflplusplus_um_random/fuzzer.py index 1a539989d..ab8ad04f0 100644 --- a/fuzzers/aflplusplus_um_random/fuzzer.py +++ b/fuzzers/aflplusplus_um_random/fuzzer.py @@ -154,11 +154,12 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.system(f"rm -rf {out}/*") aflplusplus_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") + f"{mutate_bins}/{new_fuzz_target}") shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{new_fuzz_target}") num_non_buggy += 1 @@ -196,7 +197,8 @@ def fuzz(input_corpus, output_corpus, target_binary): for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - with utils.restore_directory(input_corpus), utils.restore_directory(output_corpus): + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): try: with time_limit(timeout): aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) From 5e2f507a6f978eb7e776ce0b60aca9f4dabd939c Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Tue, 4 Oct 2022 13:00:09 -0400 Subject: [PATCH 09/18] Making build 3 hours --- fuzzers/aflplusplus_um_random/fuzzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzers/aflplusplus_um_random/fuzzer.py b/fuzzers/aflplusplus_um_random/fuzzer.py index ab8ad04f0..30c4ac521 100644 --- a/fuzzers/aflplusplus_um_random/fuzzer.py +++ b/fuzzers/aflplusplus_um_random/fuzzer.py @@ -41,7 +41,7 @@ class TimeoutException(Exception): TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours -TOTAL_BUILD_TIME = 43200 # 12 hours +TOTAL_BUILD_TIME = 10800 # 3 hours FUZZ_PROP = 0.5 DEFAULT_MUTANT_TIMEOUT = 300 GRACE_TIME = 3600 # 1 hour in seconds From 1c6de3000a3218d74b133b6b2cdc1ba562c64d82 Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Tue, 4 Oct 2022 13:37:50 -0400 Subject: [PATCH 10/18] Adding configs for 6-12 hours --- .github/workflows/fuzzers.yml | 2 + .../builder.Dockerfile | 40 ++++ .../aflplusplus_um_random_12/description.md | 10 + fuzzers/aflplusplus_um_random_12/fuzzer.py | 212 ++++++++++++++++++ .../runner.Dockerfile | 23 ++ .../builder.Dockerfile | 40 ++++ .../aflplusplus_um_random_6/description.md | 10 + fuzzers/aflplusplus_um_random_6/fuzzer.py | 212 ++++++++++++++++++ .../aflplusplus_um_random_6/runner.Dockerfile | 23 ++ service/experiment-requests.yaml | 7 + 10 files changed, 579 insertions(+) create mode 100644 fuzzers/aflplusplus_um_random_12/builder.Dockerfile create mode 100644 fuzzers/aflplusplus_um_random_12/description.md create mode 100644 fuzzers/aflplusplus_um_random_12/fuzzer.py create mode 100644 fuzzers/aflplusplus_um_random_12/runner.Dockerfile create mode 100644 fuzzers/aflplusplus_um_random_6/builder.Dockerfile create mode 100644 fuzzers/aflplusplus_um_random_6/description.md create mode 100644 fuzzers/aflplusplus_um_random_6/fuzzer.py create mode 100644 fuzzers/aflplusplus_um_random_6/runner.Dockerfile diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml index 1d3cb2b54..528f16ad2 100644 --- a/.github/workflows/fuzzers.yml +++ b/.github/workflows/fuzzers.yml @@ -27,6 +27,8 @@ jobs: - aflplusplus_tracepc - aflplusplus_um_prioritize - aflplusplus_um_random + - aflplusplus_um_random_6 + - aflplusplus_um_random_12 - aflplusplus_um_parallel - aflsmart - centipede diff --git a/fuzzers/aflplusplus_um_random_12/builder.Dockerfile b/fuzzers/aflplusplus_um_random_12/builder.Dockerfile new file mode 100644 index 000000000..abd77021b --- /dev/null +++ b/fuzzers/aflplusplus_um_random_12/builder.Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Install libstdc++ to use llvm_mode. +RUN apt-get update && \ + apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \ + libglib2.0-dev libpixman-1-dev python3-setuptools unzip \ + apt-utils apt-transport-https ca-certificates + +# Download and compile afl++. +RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \ + cd /afl && \ + git checkout b847e0f414e7b310e1a68bc501d4e2453bfce70e + +# Build without Python support as we don't need it. +# Set AFL_NO_X86 to skip flaky tests. +RUN cd /afl && unset CFLAGS && unset CXXFLAGS && \ + export CC=clang && export AFL_NO_X86=1 && \ + PYTHON_INCLUDE=/ make && make install && \ + make -C utils/aflpp_driver && \ + cp utils/aflpp_driver/libAFLDriver.a / + diff --git a/fuzzers/aflplusplus_um_random_12/description.md b/fuzzers/aflplusplus_um_random_12/description.md new file mode 100644 index 000000000..7a1c00d4d --- /dev/null +++ b/fuzzers/aflplusplus_um_random_12/description.md @@ -0,0 +1,10 @@ +# aflplusplus UM (random) - 12 hours + +Run aflplusplus over mutated code without UM prioritization. Randomly sample +list of generated mutants. + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/aflplusplus_um_random_12/fuzzer.py b/fuzzers/aflplusplus_um_random_12/fuzzer.py new file mode 100644 index 000000000..ab8ad04f0 --- /dev/null +++ b/fuzzers/aflplusplus_um_random_12/fuzzer.py @@ -0,0 +1,212 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for AFLplusplus fuzzer.""" + +# This optimized afl++ variant should always be run together with +# "aflplusplus" to show the difference - a default configured afl++ vs. +# a hand-crafted optimized one. afl++ is configured not to enable the good +# stuff by default to be as close to vanilla afl as possible. +# But this means that the good stuff is hidden away in this benchmark +# otherwise. + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.aflplusplus import fuzzer as aflplusplus_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 43200 # 12 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds +MAX_MUTANTS = 200000 + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + aflplusplus_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + random.shuffle(source_files) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + if len(mutants) > MAX_MUTANTS: + break + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + with utils.restore_directory(src), utils.restore_directory(work): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.makedirs(input_corpus_dir, exist_ok=True) + os.environ['AFL_SKIP_CRASHES'] = "1" + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/aflplusplus_um_random_12/runner.Dockerfile b/fuzzers/aflplusplus_um_random_12/runner.Dockerfile new file mode 100644 index 000000000..7aa1da8e4 --- /dev/null +++ b/fuzzers/aflplusplus_um_random_12/runner.Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# This makes interactive docker runs painless: +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out" +#ENV AFL_MAP_SIZE=2621440 +ENV PATH="$PATH:/out" +ENV AFL_SKIP_CPUFREQ=1 +ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 +ENV AFL_TESTCACHE_SIZE=2 diff --git a/fuzzers/aflplusplus_um_random_6/builder.Dockerfile b/fuzzers/aflplusplus_um_random_6/builder.Dockerfile new file mode 100644 index 000000000..abd77021b --- /dev/null +++ b/fuzzers/aflplusplus_um_random_6/builder.Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG parent_image +FROM $parent_image + +RUN apt-get update && apt-get install -y python3 +RUN pip3 install --upgrade --force pip +RUN pip install universalmutator + +# Install libstdc++ to use llvm_mode. +RUN apt-get update && \ + apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \ + libglib2.0-dev libpixman-1-dev python3-setuptools unzip \ + apt-utils apt-transport-https ca-certificates + +# Download and compile afl++. +RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \ + cd /afl && \ + git checkout b847e0f414e7b310e1a68bc501d4e2453bfce70e + +# Build without Python support as we don't need it. +# Set AFL_NO_X86 to skip flaky tests. +RUN cd /afl && unset CFLAGS && unset CXXFLAGS && \ + export CC=clang && export AFL_NO_X86=1 && \ + PYTHON_INCLUDE=/ make && make install && \ + make -C utils/aflpp_driver && \ + cp utils/aflpp_driver/libAFLDriver.a / + diff --git a/fuzzers/aflplusplus_um_random_6/description.md b/fuzzers/aflplusplus_um_random_6/description.md new file mode 100644 index 000000000..1860e6be1 --- /dev/null +++ b/fuzzers/aflplusplus_um_random_6/description.md @@ -0,0 +1,10 @@ +# aflplusplus UM (random) - 6 hours + +Run aflplusplus over mutated code without UM prioritization. Randomly sample +list of generated mutants. + +NOTE: This only works with C or C++ benchmarks. + +[builder.Dockerfile](builder.Dockerfile) +[fuzzer.py](fuzzer.py) +[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/aflplusplus_um_random_6/fuzzer.py b/fuzzers/aflplusplus_um_random_6/fuzzer.py new file mode 100644 index 000000000..e519c9a24 --- /dev/null +++ b/fuzzers/aflplusplus_um_random_6/fuzzer.py @@ -0,0 +1,212 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Integration code for AFLplusplus fuzzer.""" + +# This optimized afl++ variant should always be run together with +# "aflplusplus" to show the difference - a default configured afl++ vs. +# a hand-crafted optimized one. afl++ is configured not to enable the good +# stuff by default to be as close to vanilla afl as possible. +# But this means that the good stuff is hidden away in this benchmark +# otherwise. + +import glob +import os +from pathlib import Path +import random +import shutil +import filecmp +from subprocess import CalledProcessError +import time +import signal +import math +from contextlib import contextmanager + +from fuzzers.aflplusplus import fuzzer as aflplusplus_fuzzer +from fuzzers import utils + + +class TimeoutException(Exception): + """"Exception thrown when timeouts occur""" + + +TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours +TOTAL_BUILD_TIME = 21600 # 6 hours +FUZZ_PROP = 0.5 +DEFAULT_MUTANT_TIMEOUT = 300 +GRACE_TIME = 3600 # 1 hour in seconds +MAX_MUTANTS = 200000 + + +@contextmanager +def time_limit(seconds): + """Method to define a time limit before throwing exception""" + + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.signal(signal.SIGALRM, signal_handler) + signal.alarm(seconds) + try: + yield + finally: + signal.alarm(0) + + +def build(): # pylint: disable=too-many-locals,too-many-statements + """Build benchmark.""" + start_time = time.time() + + out = os.getenv("OUT") + src = os.getenv("SRC") + work = os.getenv("WORK") + storage_dir = "/storage" + os.mkdir(storage_dir) + mutate_dir = f"{storage_dir}/mutant_files" + os.mkdir(mutate_dir) + mutate_bins = f"{storage_dir}/mutant_bins" + os.mkdir(mutate_bins) + mutate_scripts = f"{storage_dir}/mutant_scripts" + os.mkdir(mutate_scripts) + + orig_fuzz_target = os.getenv("FUZZ_TARGET") + with utils.restore_directory(src), utils.restore_directory(work): + aflplusplus_fuzzer.build() + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{orig_fuzz_target}") + benchmark = os.getenv("BENCHMARK") + + source_extensions = [".c", ".cc", ".cpp"] + # Use heuristic to try to find benchmark directory, + # otherwise look for all files in the current directory. + subdirs = [ + name for name in os.listdir(src) + if os.path.isdir(os.path.join(src, name)) + ] + benchmark_src_dir = src + for directory in subdirs: + if directory in benchmark: + benchmark_src_dir = os.path.join(src, directory) + break + + source_files = [] + for extension in source_extensions: + source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", + recursive=True) + random.shuffle(source_files) + + mutants = [] + for source_file in source_files: + source_dir = os.path.dirname(source_file).split(src, 1)[1] + Path(f"{mutate_dir}/{source_dir}").mkdir(parents=True, exist_ok=True) + os.system(f"mutate {source_file} --mutantDir \ + {mutate_dir}/{source_dir} --noCheck > /dev/null") + source_base = os.path.basename(source_file).split(".")[0] + mutants_glob = glob.glob( + f"{mutate_dir}/{source_dir}/{source_base}.mutant.*") + mutants += [ + f"{source_dir}/{mutant.split('/')[-1]}"[1:] + for mutant in mutants_glob + ] + + if len(mutants) > MAX_MUTANTS: + break + + random.shuffle(mutants) + with open(f"{mutate_dir}/mutants.txt", "w", encoding="utf-8") as f_name: + f_name.writelines(f"{l}\n" for l in mutants) + + curr_time = time.time() + + # Add grace time for final build at end + remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - + GRACE_TIME) + with utils.restore_directory(src), utils.restore_directory(work): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + os.system(f"cp {mutate_bins}/* {out}/") + + +def fuzz(input_corpus, output_corpus, target_binary): + """Run fuzzer.""" + total_fuzzing_time = int( + os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) + total_mutant_time = int(FUZZ_PROP * total_fuzzing_time) + + mutants = glob.glob(f"{target_binary}.*") + random.shuffle(mutants) + timeout = max(DEFAULT_MUTANT_TIMEOUT, + int(total_mutant_time / max(len(mutants), 1))) + num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) + + input_corpus_dir = "/storage/input_corpus" + os.makedirs(input_corpus_dir, exist_ok=True) + os.environ['AFL_SKIP_CRASHES'] = "1" + + for mutant in mutants[:num_mutants]: + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + + os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/aflplusplus_um_random_6/runner.Dockerfile b/fuzzers/aflplusplus_um_random_6/runner.Dockerfile new file mode 100644 index 000000000..7aa1da8e4 --- /dev/null +++ b/fuzzers/aflplusplus_um_random_6/runner.Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/fuzzbench/base-image + +# This makes interactive docker runs painless: +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out" +#ENV AFL_MAP_SIZE=2621440 +ENV PATH="$PATH:/out" +ENV AFL_SKIP_CPUFREQ=1 +ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 +ENV AFL_TESTCACHE_SIZE=2 diff --git a/service/experiment-requests.yaml b/service/experiment-requests.yaml index 7ba005a9d..7ae20af35 100644 --- a/service/experiment-requests.yaml +++ b/service/experiment-requests.yaml @@ -20,6 +20,13 @@ # Please add new experiment requests towards the top of this file. # +- experiment: 2022-10-04-um-6-12 + description: "UM fuzzer experiment" + fuzzers: + - aflplusplus + - aflplusplus_um_random_6 + - aflplusplus_um_random_12 + - experiment: 2022-10-04-um description: "UM fuzzer experiment" fuzzers: From f5a4788fc14efa2478788e12ba797b1ce880c6be Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Wed, 5 Oct 2022 09:46:49 -0400 Subject: [PATCH 11/18] Adding 3 hour fuzzer, creating new experiment, fixing bug --- .github/workflows/fuzzers.yml | 2 +- benchmarks/go-ethereum_fuzzvmruntime/benchmark.yaml | 7 +++++++ benchmarks/libxslt_xpath/benchmark.yaml | 7 +++++++ fuzzers/aflplusplus_um_random/fuzzer.py | 6 ++++-- .../builder.Dockerfile | 0 .../description.md | 2 +- .../fuzzer.py | 2 +- .../runner.Dockerfile | 0 fuzzers/utils.py | 4 ++-- service/experiment-requests.yaml | 6 ++++++ 10 files changed, 29 insertions(+), 7 deletions(-) rename fuzzers/{aflplusplus_um_random_12 => aflplusplus_um_random_3}/builder.Dockerfile (100%) rename fuzzers/{aflplusplus_um_random_12 => aflplusplus_um_random_3}/description.md (87%) rename fuzzers/{aflplusplus_um_random_12 => aflplusplus_um_random_3}/fuzzer.py (99%) rename fuzzers/{aflplusplus_um_random_12 => aflplusplus_um_random_3}/runner.Dockerfile (100%) diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml index 528f16ad2..bfa7f9294 100644 --- a/.github/workflows/fuzzers.yml +++ b/.github/workflows/fuzzers.yml @@ -27,8 +27,8 @@ jobs: - aflplusplus_tracepc - aflplusplus_um_prioritize - aflplusplus_um_random + - aflplusplus_um_random_3 - aflplusplus_um_random_6 - - aflplusplus_um_random_12 - aflplusplus_um_parallel - aflsmart - centipede diff --git a/benchmarks/go-ethereum_fuzzvmruntime/benchmark.yaml b/benchmarks/go-ethereum_fuzzvmruntime/benchmark.yaml index cc0d852d4..e7991ff98 100644 --- a/benchmarks/go-ethereum_fuzzvmruntime/benchmark.yaml +++ b/benchmarks/go-ethereum_fuzzvmruntime/benchmark.yaml @@ -5,13 +5,20 @@ project: go-ethereum language: go unsupported_fuzzers: - aflplusplus_um_random + - aflplusplus_um_random_3 + - aflplusplus_um_random_6 - aflplusplus_um_prioritize + - aflplusplus_um_parallel - afl_um_random - afl_um_prioritize + - afl_um_parallel - honggfuzz_um_random - honggfuzz_um_prioritize + - honggfuzz_um_parallel - eclipser_um_random - eclipser_um_prioritize + - eclipser_um_parallel - libfuzzer_um_random - libfuzzer_um_prioritize + - libfuzzer_um_parallel - wingfuzz diff --git a/benchmarks/libxslt_xpath/benchmark.yaml b/benchmarks/libxslt_xpath/benchmark.yaml index 0e0d99a7c..48aed7a9a 100644 --- a/benchmarks/libxslt_xpath/benchmark.yaml +++ b/benchmarks/libxslt_xpath/benchmark.yaml @@ -6,15 +6,22 @@ unsupported_fuzzers: - aflcc - aflplusplus_qemu - aflplusplus_um_random + - aflplusplus_um_random_3 + - aflplusplus_um_random_6 - aflplusplus_um_prioritize + - aflplusplus_um_parallel - afl_um_random - afl_um_prioritize + - afl_um_parallel - honggfuzz_um_random - honggfuzz_um_prioritize + - honggfuzz_um_parallel - eclipser_um_random - eclipser_um_prioritize + - eclipser_um_parallel - libfuzzer_um_random - libfuzzer_um_prioritize + - libfuzzer_um_parallel - aflplusplus_qemu_tracepc - aflplusplus_frida - klee diff --git a/fuzzers/aflplusplus_um_random/fuzzer.py b/fuzzers/aflplusplus_um_random/fuzzer.py index 30c4ac521..7f4bc1bfd 100644 --- a/fuzzers/aflplusplus_um_random/fuzzer.py +++ b/fuzzers/aflplusplus_um_random/fuzzer.py @@ -41,7 +41,7 @@ class TimeoutException(Exception): TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours -TOTAL_BUILD_TIME = 10800 # 3 hours +TOTAL_BUILD_TIME = 43200 # 12 hours FUZZ_PROP = 0.5 DEFAULT_MUTANT_TIMEOUT = 300 GRACE_TIME = 3600 # 1 hour in seconds @@ -131,7 +131,9 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - with utils.restore_directory(src), utils.restore_directory(work): + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): try: with time_limit(remaining_time): num_non_buggy = 1 diff --git a/fuzzers/aflplusplus_um_random_12/builder.Dockerfile b/fuzzers/aflplusplus_um_random_3/builder.Dockerfile similarity index 100% rename from fuzzers/aflplusplus_um_random_12/builder.Dockerfile rename to fuzzers/aflplusplus_um_random_3/builder.Dockerfile diff --git a/fuzzers/aflplusplus_um_random_12/description.md b/fuzzers/aflplusplus_um_random_3/description.md similarity index 87% rename from fuzzers/aflplusplus_um_random_12/description.md rename to fuzzers/aflplusplus_um_random_3/description.md index 7a1c00d4d..a5ed5a4c7 100644 --- a/fuzzers/aflplusplus_um_random_12/description.md +++ b/fuzzers/aflplusplus_um_random_3/description.md @@ -1,4 +1,4 @@ -# aflplusplus UM (random) - 12 hours +# aflplusplus UM (random) - 3 hours Run aflplusplus over mutated code without UM prioritization. Randomly sample list of generated mutants. diff --git a/fuzzers/aflplusplus_um_random_12/fuzzer.py b/fuzzers/aflplusplus_um_random_3/fuzzer.py similarity index 99% rename from fuzzers/aflplusplus_um_random_12/fuzzer.py rename to fuzzers/aflplusplus_um_random_3/fuzzer.py index ab8ad04f0..30c4ac521 100644 --- a/fuzzers/aflplusplus_um_random_12/fuzzer.py +++ b/fuzzers/aflplusplus_um_random_3/fuzzer.py @@ -41,7 +41,7 @@ class TimeoutException(Exception): TOTAL_FUZZING_TIME_DEFAULT = 82800 # 23 hours -TOTAL_BUILD_TIME = 43200 # 12 hours +TOTAL_BUILD_TIME = 10800 # 3 hours FUZZ_PROP = 0.5 DEFAULT_MUTANT_TIMEOUT = 300 GRACE_TIME = 3600 # 1 hour in seconds diff --git a/fuzzers/aflplusplus_um_random_12/runner.Dockerfile b/fuzzers/aflplusplus_um_random_3/runner.Dockerfile similarity index 100% rename from fuzzers/aflplusplus_um_random_12/runner.Dockerfile rename to fuzzers/aflplusplus_um_random_3/runner.Dockerfile diff --git a/fuzzers/utils.py b/fuzzers/utils.py index 38fd97cbd..0a2bca0e6 100644 --- a/fuzzers/utils.py +++ b/fuzzers/utils.py @@ -102,7 +102,7 @@ def get_config_value(attribute): @contextlib.contextmanager -def restore_directory(directory): +def restore_directory(directory, ignore_errors=False): """Helper contextmanager that when created saves a backup of |directory| and when closed/exited replaces |directory| with the backup. @@ -127,7 +127,7 @@ def restore_directory(directory): backup = os.path.join(temp_dir, os.path.basename(directory)) shutil.copytree(directory, backup, symlinks=True) yield - shutil.rmtree(directory) + shutil.rmtree(directory, ignore_errors=ignore_errors) shutil.move(backup, directory) try: os.getcwd() diff --git a/service/experiment-requests.yaml b/service/experiment-requests.yaml index 7ae20af35..69290df89 100644 --- a/service/experiment-requests.yaml +++ b/service/experiment-requests.yaml @@ -20,6 +20,12 @@ # Please add new experiment requests towards the top of this file. # +- experiment: 2022-10-05-um-3 + description: "UM fuzzer experiment" + fuzzers: + - aflplusplus + - aflplusplus_um_random_3 + - experiment: 2022-10-04-um-6-12 description: "UM fuzzer experiment" fuzzers: From 546c17cbb07e4e045b8b77739cc0f3c24246d161 Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Wed, 5 Oct 2022 09:56:50 -0400 Subject: [PATCH 12/18] Make 3/6 fuzzers up to date too --- fuzzers/aflplusplus_um_random_3/fuzzer.py | 4 +++- fuzzers/aflplusplus_um_random_6/fuzzer.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fuzzers/aflplusplus_um_random_3/fuzzer.py b/fuzzers/aflplusplus_um_random_3/fuzzer.py index 30c4ac521..ed56f8a7a 100644 --- a/fuzzers/aflplusplus_um_random_3/fuzzer.py +++ b/fuzzers/aflplusplus_um_random_3/fuzzer.py @@ -131,7 +131,9 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - with utils.restore_directory(src), utils.restore_directory(work): + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): try: with time_limit(remaining_time): num_non_buggy = 1 diff --git a/fuzzers/aflplusplus_um_random_6/fuzzer.py b/fuzzers/aflplusplus_um_random_6/fuzzer.py index e519c9a24..2f6f32273 100644 --- a/fuzzers/aflplusplus_um_random_6/fuzzer.py +++ b/fuzzers/aflplusplus_um_random_6/fuzzer.py @@ -131,7 +131,9 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - with utils.restore_directory(src), utils.restore_directory(work): + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): try: with time_limit(remaining_time): num_non_buggy = 1 From a5b22f8f134078f36d2db9308325fb88c7cbccb1 Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Thu, 6 Oct 2022 10:16:16 -0400 Subject: [PATCH 13/18] Fix all fuzzers --- fuzzers/afl_um_parallel/fuzzer.py | 103 +++++++++--------- fuzzers/afl_um_prioritize/fuzzer.py | 109 +++++++++++--------- fuzzers/afl_um_random/fuzzer.py | 103 +++++++++--------- fuzzers/aflplusplus_um_parallel/fuzzer.py | 103 +++++++++--------- fuzzers/aflplusplus_um_prioritize/fuzzer.py | 109 +++++++++++--------- fuzzers/eclipser_um_parallel/fuzzer.py | 104 ++++++++++--------- fuzzers/eclipser_um_prioritize/fuzzer.py | 109 +++++++++++--------- fuzzers/eclipser_um_random/fuzzer.py | 103 +++++++++--------- fuzzers/honggfuzz_um_parallel/fuzzer.py | 103 +++++++++--------- fuzzers/honggfuzz_um_prioritize/fuzzer.py | 109 +++++++++++--------- fuzzers/honggfuzz_um_random/fuzzer.py | 103 +++++++++--------- fuzzers/libfuzzer_um_parallel/fuzzer.py | 103 +++++++++--------- fuzzers/libfuzzer_um_prioritize/fuzzer.py | 109 +++++++++++--------- fuzzers/libfuzzer_um_random/fuzzer.py | 103 +++++++++--------- 14 files changed, 785 insertions(+), 688 deletions(-) diff --git a/fuzzers/afl_um_parallel/fuzzer.py b/fuzzers/afl_um_parallel/fuzzer.py index 2541906e5..b5f8b777e 100644 --- a/fuzzers/afl_um_parallel/fuzzer.py +++ b/fuzzers/afl_um_parallel/fuzzer.py @@ -125,46 +125,50 @@ def build(): # pylint: disable=too-many-locals,too-many-statements remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ - .{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - afl_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") afl_fuzzer.build() @@ -184,17 +188,20 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) os.environ['AFL_SKIP_CRASHES'] = "1" for mutant in mutants[:num_mutants]: - os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - afl_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + afl_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") afl_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/afl_um_prioritize/fuzzer.py b/fuzzers/afl_um_prioritize/fuzzer.py index b68be11b5..0af39ae4a 100755 --- a/fuzzers/afl_um_prioritize/fuzzer.py +++ b/fuzzers/afl_um_prioritize/fuzzer.py @@ -160,49 +160,53 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(order): - with utils.restore_directory(src), utils.restore_directory( - work): - key, line = order[ind] - mutant = prioritize_map[key][line] - print(mutant) - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ - .{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - afl_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - print( - f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") - else: - print(f"EQUAL {num_non_buggy}, ind: {ind}") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, \ + ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") afl_fuzzer.build() @@ -222,18 +226,21 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) os.environ['AFL_SKIP_CRASHES'] = "1" for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - afl_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + afl_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") afl_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/afl_um_random/fuzzer.py b/fuzzers/afl_um_random/fuzzer.py index 7f6d5a5c7..4dde44e27 100644 --- a/fuzzers/afl_um_random/fuzzer.py +++ b/fuzzers/afl_um_random/fuzzer.py @@ -125,46 +125,50 @@ def build(): # pylint: disable=too-many-locals,too-many-statements remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ - .{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - afl_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") afl_fuzzer.build() @@ -184,18 +188,21 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) os.environ['AFL_SKIP_CRASHES'] = "1" for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - afl_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + afl_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") afl_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/aflplusplus_um_parallel/fuzzer.py b/fuzzers/aflplusplus_um_parallel/fuzzer.py index 91bcf8c40..57efc54d3 100644 --- a/fuzzers/aflplusplus_um_parallel/fuzzer.py +++ b/fuzzers/aflplusplus_um_parallel/fuzzer.py @@ -132,46 +132,50 @@ def build(): # pylint: disable=too-many-locals,too-many-statements remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") aflplusplus_fuzzer.build() @@ -191,17 +195,20 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) os.environ['AFL_SKIP_CRASHES'] = "1" for mutant in mutants[:num_mutants]: - try: - with time_limit(timeout): - aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/aflplusplus_um_prioritize/fuzzer.py b/fuzzers/aflplusplus_um_prioritize/fuzzer.py index d2b2ed4ce..02498f75f 100755 --- a/fuzzers/aflplusplus_um_prioritize/fuzzer.py +++ b/fuzzers/aflplusplus_um_prioritize/fuzzer.py @@ -167,49 +167,53 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(order): - with utils.restore_directory(src), utils.restore_directory( - work): - key, line = order[ind] - mutant = prioritize_map[key][line] - print(mutant) - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - print( - f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") - else: - print(f"EQUAL {num_non_buggy}, ind: {ind}") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, \ + ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") aflplusplus_fuzzer.build() @@ -229,18 +233,21 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) os.environ['AFL_SKIP_CRASHES'] = "1" for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") aflplusplus_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/eclipser_um_parallel/fuzzer.py b/fuzzers/eclipser_um_parallel/fuzzer.py index ef09a361b..28244dbcd 100644 --- a/fuzzers/eclipser_um_parallel/fuzzer.py +++ b/fuzzers/eclipser_um_parallel/fuzzer.py @@ -124,47 +124,50 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - eclipser_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") eclipser_fuzzer.build() @@ -184,16 +187,19 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) for mutant in mutants[:num_mutants]: - os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - eclipser_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + eclipser_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") eclipser_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/eclipser_um_prioritize/fuzzer.py b/fuzzers/eclipser_um_prioritize/fuzzer.py index e1a626f76..8a69906e4 100755 --- a/fuzzers/eclipser_um_prioritize/fuzzer.py +++ b/fuzzers/eclipser_um_prioritize/fuzzer.py @@ -160,49 +160,53 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(order): - with utils.restore_directory(src), utils.restore_directory( - work): - key, line = order[ind] - mutant = prioritize_map[key][line] - print(mutant) - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - eclipser_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - print( - f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") - else: - print(f"EQUAL {num_non_buggy}, ind: {ind}") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, \ + ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") eclipser_fuzzer.build() @@ -222,17 +226,20 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - eclipser_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + eclipser_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") eclipser_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/eclipser_um_random/fuzzer.py b/fuzzers/eclipser_um_random/fuzzer.py index 72b93e325..4050794a3 100644 --- a/fuzzers/eclipser_um_random/fuzzer.py +++ b/fuzzers/eclipser_um_random/fuzzer.py @@ -125,46 +125,50 @@ def build(): # pylint: disable=too-many-locals,too-many-statements remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - eclipser_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") eclipser_fuzzer.build() @@ -184,17 +188,20 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - eclipser_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + eclipser_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") eclipser_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/honggfuzz_um_parallel/fuzzer.py b/fuzzers/honggfuzz_um_parallel/fuzzer.py index 38e9ccb43..98a69588d 100644 --- a/fuzzers/honggfuzz_um_parallel/fuzzer.py +++ b/fuzzers/honggfuzz_um_parallel/fuzzer.py @@ -125,46 +125,50 @@ def build(): # pylint: disable=too-many-locals,too-many-statements remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - honggfuzz_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") honggfuzz_fuzzer.build() @@ -184,17 +188,20 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) os.environ['AFL_SKIP_CRASHES'] = "1" for mutant in mutants[:num_mutants]: - os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/honggfuzz_um_prioritize/fuzzer.py b/fuzzers/honggfuzz_um_prioritize/fuzzer.py index 16676a0ca..361d15570 100755 --- a/fuzzers/honggfuzz_um_prioritize/fuzzer.py +++ b/fuzzers/honggfuzz_um_prioritize/fuzzer.py @@ -160,49 +160,53 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(order): - with utils.restore_directory(src), utils.restore_directory( - work): - key, line = order[ind] - mutant = prioritize_map[key][line] - print(mutant) - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - honggfuzz_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - print( - f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") - else: - print(f"EQUAL {num_non_buggy}, ind: {ind}") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, \ + ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") honggfuzz_fuzzer.build() @@ -222,17 +226,20 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/honggfuzz_um_random/fuzzer.py b/fuzzers/honggfuzz_um_random/fuzzer.py index 8abdc90e0..6a664fbc8 100644 --- a/fuzzers/honggfuzz_um_random/fuzzer.py +++ b/fuzzers/honggfuzz_um_random/fuzzer.py @@ -125,46 +125,50 @@ def build(): # pylint: disable=too-many-locals,too-many-statements remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - honggfuzz_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") honggfuzz_fuzzer.build() @@ -184,18 +188,21 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) os.environ['AFL_SKIP_CRASHES'] = "1" for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") honggfuzz_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/libfuzzer_um_parallel/fuzzer.py b/fuzzers/libfuzzer_um_parallel/fuzzer.py index d0959f3e5..8f827c7de 100755 --- a/fuzzers/libfuzzer_um_parallel/fuzzer.py +++ b/fuzzers/libfuzzer_um_parallel/fuzzer.py @@ -125,46 +125,50 @@ def build(): # pylint: disable=too-many-locals,too-many-statements remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - libfuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + libfuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") libfuzzer.build() @@ -184,16 +188,19 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) for mutant in mutants[:num_mutants]: - os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - libfuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + libfuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") libfuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/libfuzzer_um_prioritize/fuzzer.py b/fuzzers/libfuzzer_um_prioritize/fuzzer.py index a13d6eae1..51769a423 100755 --- a/fuzzers/libfuzzer_um_prioritize/fuzzer.py +++ b/fuzzers/libfuzzer_um_prioritize/fuzzer.py @@ -160,49 +160,53 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(order): - with utils.restore_directory(src), utils.restore_directory( - work): - key, line = order[ind] - mutant = prioritize_map[key][line] - print(mutant) - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - libfuzzer_fuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - print( - f"FOUND NOT EQUAL {num_non_buggy}, ind: {ind}") - else: - print(f"EQUAL {num_non_buggy}, ind: {ind}") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + libfuzzer_fuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, \ + ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") libfuzzer_fuzzer.build() @@ -222,17 +226,20 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - libfuzzer_fuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + libfuzzer_fuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") libfuzzer_fuzzer.fuzz(input_corpus, output_corpus, target_binary) diff --git a/fuzzers/libfuzzer_um_random/fuzzer.py b/fuzzers/libfuzzer_um_random/fuzzer.py index 2a6fc886a..3864c6073 100755 --- a/fuzzers/libfuzzer_um_random/fuzzer.py +++ b/fuzzers/libfuzzer_um_random/fuzzer.py @@ -125,46 +125,50 @@ def build(): # pylint: disable=too-many-locals,too-many-statements remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - libfuzzer.build() - if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + with utils.restore_directory(src, + ignore_errors=True), utils.restore_directory( + work, ignore_errors=True): + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + libfuzzer.build() + if not filecmp.cmp( + f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") libfuzzer.build() @@ -184,17 +188,20 @@ def fuzz(input_corpus, output_corpus, target_binary): num_mutants = min(math.ceil(total_mutant_time / timeout), len(mutants)) input_corpus_dir = "/storage/input_corpus" - os.mkdir("/storage") - os.mkdir(input_corpus_dir) + os.makedirs(input_corpus_dir, exist_ok=True) for mutant in mutants[:num_mutants]: os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") - try: - with time_limit(timeout): - libfuzzer.fuzz(input_corpus, output_corpus, mutant) - except TimeoutException: - pass - os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") + with utils.restore_directory(input_corpus), utils.restore_directory( + output_corpus): + try: + with time_limit(timeout): + libfuzzer.fuzz(input_corpus, output_corpus, mutant) + except TimeoutException: + pass + except CalledProcessError: + pass + os.system(f"cp -r {output_corpus}/* {input_corpus_dir}/*") os.system(f"cp -r {input_corpus_dir}/* {input_corpus}/*") libfuzzer.fuzz(input_corpus, output_corpus, target_binary) From f1f3ece9aec9be7b2bfe144573941ef356c1eb6c Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Thu, 6 Oct 2022 11:07:27 -0400 Subject: [PATCH 14/18] One more large experiment --- service/experiment-requests.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/service/experiment-requests.yaml b/service/experiment-requests.yaml index e50eec91e..c623a34bb 100644 --- a/service/experiment-requests.yaml +++ b/service/experiment-requests.yaml @@ -20,6 +20,29 @@ # Please add new experiment requests towards the top of this file. # +- experiment: 2022-10-06-um-full + description: "UM fuzzer experiment" + fuzzers: + - aflplusplus + - aflplusplus_um_parallel + - aflplusplus + - libfuzzer_um_prioritize + - libfuzzer_um_random + - libfuzzer_um_parallel + - libfuzzer + - afl_um_prioritize + - afl_um_random + - afl_um_parallel + - afl + - eclipser_um_prioritize + - eclipser_um_random + - eclipser_um_parallel + - eclipser + - honggfuzz_um_prioritize + - honggfuzz_um_random + - honggfuzz_um_parallel + - honggfuzz + - experiment: 2022-10-05-um-3 description: "UM fuzzer experiment" fuzzers: From 1a42eb50f85e8c57ad6b1c4796ec6193b8d7595f Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Thu, 6 Oct 2022 11:09:41 -0400 Subject: [PATCH 15/18] Merging --- .../go-ethereum_fuzzvmruntime/Dockerfile | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 benchmarks/go-ethereum_fuzzvmruntime/Dockerfile diff --git a/benchmarks/go-ethereum_fuzzvmruntime/Dockerfile b/benchmarks/go-ethereum_fuzzvmruntime/Dockerfile deleted file mode 100644 index 8d903780c..000000000 --- a/benchmarks/go-ethereum_fuzzvmruntime/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2020 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -FROM gcr.io/oss-fuzz-base/base-builder@sha256:aeb14ce0acc046345fd42cc78af7a9557bfebde4af29012b6cbac1f0398adc15 - -RUN git clone --single-branch --depth=1 https://github.com/ethereum/go-ethereum $GOPATH/src/github.com/ethereum/go-ethereum -RUN (cd $GOPATH/src/github.com/ethereum/go-ethereum && go mod download) - -RUN cp $GOPATH/src/github.com/ethereum/go-ethereum/oss-fuzz.sh $SRC/build.sh -# Enable this for easier local testing / repro -#ADD build.sh $SRC/build.sh - -WORKDIR $SRC/ - From 6c814bd741b9fe2c336fcfa7f2cfa1bf8afb02da Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Thu, 6 Oct 2022 11:25:50 -0400 Subject: [PATCH 16/18] Adding line back --- fuzzers/libfuzzer_um_random/fuzzer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fuzzers/libfuzzer_um_random/fuzzer.py b/fuzzers/libfuzzer_um_random/fuzzer.py index 6df1b4596..3864c6073 100755 --- a/fuzzers/libfuzzer_um_random/fuzzer.py +++ b/fuzzers/libfuzzer_um_random/fuzzer.py @@ -97,6 +97,7 @@ def build(): # pylint: disable=too-many-locals,too-many-statements source_files += glob.glob(f"{benchmark_src_dir}/**/*{extension}", recursive=True) random.shuffle(source_files) + mutants = [] for source_file in source_files: source_dir = os.path.dirname(source_file).split(src, 1)[1] From 791ac32efcdab4be33d696c81122d07bae580b4c Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Thu, 6 Oct 2022 11:32:24 -0400 Subject: [PATCH 17/18] Making experiments file not have duplicates --- service/experiment-requests.yaml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/service/experiment-requests.yaml b/service/experiment-requests.yaml index f1913e376..c623a34bb 100644 --- a/service/experiment-requests.yaml +++ b/service/experiment-requests.yaml @@ -19,19 +19,6 @@ # You can run "make presubmit" to do basic validation on this file. # Please add new experiment requests towards the top of this file. # -- experiment: 2022-08-31-aflpp-um - description: "afl++ um coverage experiments (compare our fuzzers against afl++)" - fuzzers: - - aflplusplus_um_prioritize - - aflplusplus_um_random - - aflplusplus - -- experiment: 2022-08-25-aflpp-um - description: "afl++ um coverage experiments (compare our fuzzers against afl++)" - fuzzers: - - aflplusplus_um_prioritize - - aflplusplus_um_random - - aflplusplus - experiment: 2022-10-06-um-full description: "UM fuzzer experiment" @@ -62,12 +49,6 @@ - aflplusplus - aflplusplus_um_random_3 -- experiment: 2022-10-05-um-3 - description: "UM fuzzer experiment" - fuzzers: - - aflplusplus - - aflplusplus_um_random_3 - - experiment: 2022-06-08-dissecting-repro description: "Dissecting AFL paper (scheduling)" type: bug From 9ef0d261a56dd5e1cd123cfe913bda57b52e1c02 Mon Sep 17 00:00:00 2001 From: Kush Jain Date: Fri, 7 Oct 2022 07:46:22 -0400 Subject: [PATCH 18/18] remove final failing build entirely --- fuzzers/afl_um_parallel/fuzzer.py | 90 ++++++++++--------- fuzzers/afl_um_prioritize/fuzzer.py | 96 ++++++++++----------- fuzzers/afl_um_random/fuzzer.py | 90 ++++++++++--------- fuzzers/aflplusplus_um_parallel/fuzzer.py | 90 ++++++++++--------- fuzzers/aflplusplus_um_prioritize/fuzzer.py | 96 ++++++++++----------- fuzzers/aflplusplus_um_random/fuzzer.py | 89 ++++++++++--------- fuzzers/aflplusplus_um_random_3/fuzzer.py | 89 ++++++++++--------- fuzzers/aflplusplus_um_random_6/fuzzer.py | 89 ++++++++++--------- fuzzers/eclipser_um_parallel/fuzzer.py | 89 ++++++++++--------- fuzzers/eclipser_um_prioritize/fuzzer.py | 96 ++++++++++----------- fuzzers/eclipser_um_random/fuzzer.py | 90 ++++++++++--------- fuzzers/honggfuzz_um_parallel/fuzzer.py | 90 ++++++++++--------- fuzzers/honggfuzz_um_prioritize/fuzzer.py | 96 ++++++++++----------- fuzzers/honggfuzz_um_random/fuzzer.py | 90 ++++++++++--------- fuzzers/libfuzzer_um_parallel/fuzzer.py | 90 ++++++++++--------- fuzzers/libfuzzer_um_prioritize/fuzzer.py | 96 ++++++++++----------- fuzzers/libfuzzer_um_random/fuzzer.py | 90 ++++++++++--------- service/experiment-requests.yaml | 29 +++++++ 18 files changed, 792 insertions(+), 793 deletions(-) diff --git a/fuzzers/afl_um_parallel/fuzzer.py b/fuzzers/afl_um_parallel/fuzzer.py index b5f8b777e..5b7cc34ee 100644 --- a/fuzzers/afl_um_parallel/fuzzer.py +++ b/fuzzers/afl_um_parallel/fuzzer.py @@ -71,12 +71,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): afl_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -124,54 +127,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ - .{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - afl_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - afl_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/afl_um_prioritize/fuzzer.py b/fuzzers/afl_um_prioritize/fuzzer.py index 0af39ae4a..123da9455 100755 --- a/fuzzers/afl_um_prioritize/fuzzer.py +++ b/fuzzers/afl_um_prioritize/fuzzer.py @@ -73,12 +73,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): afl_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") total_fuzzing_time = int( os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) @@ -159,57 +162,52 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(order): - with utils.restore_directory(src), utils.restore_directory( - work): - key, line = order[ind] - mutant = prioritize_map[key][line] - print(mutant) - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ - .{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - afl_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - print(f"FOUND NOT EQUAL {num_non_buggy}, \ - ind: {ind}") - else: - print(f"EQUAL {num_non_buggy}, ind: {ind}") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, \ + ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - afl_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/afl_um_random/fuzzer.py b/fuzzers/afl_um_random/fuzzer.py index 4dde44e27..24be49a45 100644 --- a/fuzzers/afl_um_random/fuzzer.py +++ b/fuzzers/afl_um_random/fuzzer.py @@ -71,12 +71,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): afl_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -124,54 +127,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ - .{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - afl_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}\ + .{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + afl_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - afl_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/aflplusplus_um_parallel/fuzzer.py b/fuzzers/aflplusplus_um_parallel/fuzzer.py index 57efc54d3..ea24a2bd0 100644 --- a/fuzzers/aflplusplus_um_parallel/fuzzer.py +++ b/fuzzers/aflplusplus_um_parallel/fuzzer.py @@ -78,12 +78,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): aflplusplus_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -131,54 +134,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/aflplusplus_um_prioritize/fuzzer.py b/fuzzers/aflplusplus_um_prioritize/fuzzer.py index 02498f75f..301f7b22f 100755 --- a/fuzzers/aflplusplus_um_prioritize/fuzzer.py +++ b/fuzzers/aflplusplus_um_prioritize/fuzzer.py @@ -80,12 +80,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): aflplusplus_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") total_fuzzing_time = int( os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) @@ -166,57 +169,52 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(order): - with utils.restore_directory(src), utils.restore_directory( - work): - key, line = order[ind] - mutant = prioritize_map[key][line] - print(mutant) - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - print(f"FOUND NOT EQUAL {num_non_buggy}, \ - ind: {ind}") - else: - print(f"EQUAL {num_non_buggy}, ind: {ind}") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, \ + ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/aflplusplus_um_random/fuzzer.py b/fuzzers/aflplusplus_um_random/fuzzer.py index 7f4bc1bfd..b250766aa 100644 --- a/fuzzers/aflplusplus_um_random/fuzzer.py +++ b/fuzzers/aflplusplus_um_random/fuzzer.py @@ -78,12 +78,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): aflplusplus_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -131,53 +134,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/aflplusplus_um_random_3/fuzzer.py b/fuzzers/aflplusplus_um_random_3/fuzzer.py index ed56f8a7a..f4b9a3620 100644 --- a/fuzzers/aflplusplus_um_random_3/fuzzer.py +++ b/fuzzers/aflplusplus_um_random_3/fuzzer.py @@ -78,12 +78,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): aflplusplus_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -131,53 +134,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/aflplusplus_um_random_6/fuzzer.py b/fuzzers/aflplusplus_um_random_6/fuzzer.py index 2f6f32273..1a15d8494 100644 --- a/fuzzers/aflplusplus_um_random_6/fuzzer.py +++ b/fuzzers/aflplusplus_um_random_6/fuzzer.py @@ -78,12 +78,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): aflplusplus_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -131,53 +134,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + aflplusplus_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - aflplusplus_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/eclipser_um_parallel/fuzzer.py b/fuzzers/eclipser_um_parallel/fuzzer.py index 28244dbcd..73ada3eaa 100644 --- a/fuzzers/eclipser_um_parallel/fuzzer.py +++ b/fuzzers/eclipser_um_parallel/fuzzer.py @@ -71,12 +71,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): eclipser_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -124,53 +127,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - eclipser_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - eclipser_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/eclipser_um_prioritize/fuzzer.py b/fuzzers/eclipser_um_prioritize/fuzzer.py index 8a69906e4..19324dab7 100755 --- a/fuzzers/eclipser_um_prioritize/fuzzer.py +++ b/fuzzers/eclipser_um_prioritize/fuzzer.py @@ -73,12 +73,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): eclipser_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") total_fuzzing_time = int( os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) @@ -159,57 +162,52 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(order): - with utils.restore_directory(src), utils.restore_directory( - work): - key, line = order[ind] - mutant = prioritize_map[key][line] - print(mutant) - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - eclipser_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - print(f"FOUND NOT EQUAL {num_non_buggy}, \ - ind: {ind}") - else: - print(f"EQUAL {num_non_buggy}, ind: {ind}") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, \ + ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - eclipser_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/eclipser_um_random/fuzzer.py b/fuzzers/eclipser_um_random/fuzzer.py index 4050794a3..2c943ad33 100644 --- a/fuzzers/eclipser_um_random/fuzzer.py +++ b/fuzzers/eclipser_um_random/fuzzer.py @@ -71,12 +71,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): eclipser_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -124,54 +127,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - eclipser_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + eclipser_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - eclipser_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/honggfuzz_um_parallel/fuzzer.py b/fuzzers/honggfuzz_um_parallel/fuzzer.py index 98a69588d..c17fc8ce8 100644 --- a/fuzzers/honggfuzz_um_parallel/fuzzer.py +++ b/fuzzers/honggfuzz_um_parallel/fuzzer.py @@ -71,12 +71,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): honggfuzz_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -124,54 +127,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - honggfuzz_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - honggfuzz_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/honggfuzz_um_prioritize/fuzzer.py b/fuzzers/honggfuzz_um_prioritize/fuzzer.py index 361d15570..59f86d3a7 100755 --- a/fuzzers/honggfuzz_um_prioritize/fuzzer.py +++ b/fuzzers/honggfuzz_um_prioritize/fuzzer.py @@ -73,12 +73,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): honggfuzz_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") total_fuzzing_time = int( os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) @@ -159,57 +162,52 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(order): - with utils.restore_directory(src), utils.restore_directory( - work): - key, line = order[ind] - mutant = prioritize_map[key][line] - print(mutant) - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - honggfuzz_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - print(f"FOUND NOT EQUAL {num_non_buggy}, \ - ind: {ind}") - else: - print(f"EQUAL {num_non_buggy}, ind: {ind}") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, \ + ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - honggfuzz_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/honggfuzz_um_random/fuzzer.py b/fuzzers/honggfuzz_um_random/fuzzer.py index 6a664fbc8..01b87c96c 100644 --- a/fuzzers/honggfuzz_um_random/fuzzer.py +++ b/fuzzers/honggfuzz_um_random/fuzzer.py @@ -71,12 +71,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): honggfuzz_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -124,54 +127,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - honggfuzz_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + honggfuzz_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - honggfuzz_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/libfuzzer_um_parallel/fuzzer.py b/fuzzers/libfuzzer_um_parallel/fuzzer.py index 8f827c7de..bfc616869 100755 --- a/fuzzers/libfuzzer_um_parallel/fuzzer.py +++ b/fuzzers/libfuzzer_um_parallel/fuzzer.py @@ -71,12 +71,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): libfuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -124,54 +127,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - libfuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + libfuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - libfuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/libfuzzer_um_prioritize/fuzzer.py b/fuzzers/libfuzzer_um_prioritize/fuzzer.py index 51769a423..719683305 100755 --- a/fuzzers/libfuzzer_um_prioritize/fuzzer.py +++ b/fuzzers/libfuzzer_um_prioritize/fuzzer.py @@ -73,12 +73,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): libfuzzer_fuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") total_fuzzing_time = int( os.getenv('MAX_TOTAL_TIME', str(TOTAL_FUZZING_TIME_DEFAULT))) @@ -159,57 +162,52 @@ def build(): # pylint: disable=too-many-locals,too-many-statements,too-many-bra # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(order): - with utils.restore_directory(src), utils.restore_directory( - work): - key, line = order[ind] - mutant = prioritize_map[key][line] - print(mutant) - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - libfuzzer_fuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - print(f"FOUND NOT EQUAL {num_non_buggy}, \ - ind: {ind}") - else: - print(f"EQUAL {num_non_buggy}, ind: {ind}") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(order): + with utils.restore_directory(src), utils.restore_directory( + work): + key, line = order[ind] + mutant = prioritize_map[key][line] + print(mutant) + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + libfuzzer_fuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + print(f"FOUND NOT EQUAL {num_non_buggy}, \ + ind: {ind}") + else: + print(f"EQUAL {num_non_buggy}, ind: {ind}") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - libfuzzer_fuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/fuzzers/libfuzzer_um_random/fuzzer.py b/fuzzers/libfuzzer_um_random/fuzzer.py index 3864c6073..bd0d67072 100755 --- a/fuzzers/libfuzzer_um_random/fuzzer.py +++ b/fuzzers/libfuzzer_um_random/fuzzer.py @@ -71,12 +71,15 @@ def build(): # pylint: disable=too-many-locals,too-many-statements os.mkdir(mutate_bins) mutate_scripts = f"{storage_dir}/mutant_scripts" os.mkdir(mutate_scripts) + orig_out = f"{storage_dir}/orig_out" + os.mkdir(orig_out) orig_fuzz_target = os.getenv("FUZZ_TARGET") with utils.restore_directory(src), utils.restore_directory(work): libfuzzer.build() shutil.copy(f"{out}/{orig_fuzz_target}", f"{mutate_bins}/{orig_fuzz_target}") + os.system(f"cp -r {out}/* {orig_out}/") benchmark = os.getenv("BENCHMARK") source_extensions = [".c", ".cc", ".cpp"] @@ -124,54 +127,49 @@ def build(): # pylint: disable=too-many-locals,too-many-statements # Add grace time for final build at end remaining_time = int(TOTAL_BUILD_TIME - (start_time - curr_time) - GRACE_TIME) - - with utils.restore_directory(src, - ignore_errors=True), utils.restore_directory( - work, ignore_errors=True): - try: - with time_limit(remaining_time): - num_non_buggy = 1 - ind = 0 - while ind < len(mutants): - with utils.restore_directory(src), utils.restore_directory( - work): - mutant = mutants[ind] - suffix = "." + mutant.split(".")[-1] - mpart = ".mutant." + mutant.split(".mutant.")[1] - source_file = f"{src}/{mutant.replace(mpart, suffix)}" - print(source_file) - print(f"{mutate_dir}/{mutant}") - os.system(f"cp {source_file} {mutate_dir}/orig") - os.system(f"cp {mutate_dir}/{mutant} {source_file}") - - try: - new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ - f".{num_non_buggy}" - - os.system(f"rm -rf {out}/*") - libfuzzer.build() - if not filecmp.cmp( - f'{mutate_bins}/{orig_fuzz_target}', - f'{out}/{orig_fuzz_target}', - shallow=False): - print(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - shutil.copy(f"{out}/{orig_fuzz_target}", - f"{mutate_bins}/{new_fuzz_target}") - num_non_buggy += 1 - else: - print("EQUAL") - except RuntimeError: - pass - except CalledProcessError: - pass - os.system(f"cp {mutate_dir}/orig {source_file}") - ind += 1 - except TimeoutException: - pass + try: + with time_limit(remaining_time): + num_non_buggy = 1 + ind = 0 + while ind < len(mutants): + with utils.restore_directory(src), utils.restore_directory( + work): + mutant = mutants[ind] + suffix = "." + mutant.split(".")[-1] + mpart = ".mutant." + mutant.split(".mutant.")[1] + source_file = f"{src}/{mutant.replace(mpart, suffix)}" + print(source_file) + print(f"{mutate_dir}/{mutant}") + os.system(f"cp {source_file} {mutate_dir}/orig") + os.system(f"cp {mutate_dir}/{mutant} {source_file}") + + try: + new_fuzz_target = f"{os.getenv('FUZZ_TARGET')}"\ + f".{num_non_buggy}" + + os.system(f"rm -rf {out}/*") + libfuzzer.build() + if not filecmp.cmp(f'{mutate_bins}/{orig_fuzz_target}', + f'{out}/{orig_fuzz_target}', + shallow=False): + print(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + shutil.copy(f"{out}/{orig_fuzz_target}", + f"{mutate_bins}/{new_fuzz_target}") + num_non_buggy += 1 + else: + print("EQUAL") + except RuntimeError: + pass + except CalledProcessError: + pass + os.system(f"cp {mutate_dir}/orig {source_file}") + ind += 1 + except TimeoutException: + pass os.system(f"rm -rf {out}/*") - libfuzzer.build() + os.system(f"cp -r {orig_out}/* {out}/") os.system(f"cp {mutate_bins}/* {out}/") diff --git a/service/experiment-requests.yaml b/service/experiment-requests.yaml index c623a34bb..0c0a4038f 100644 --- a/service/experiment-requests.yaml +++ b/service/experiment-requests.yaml @@ -20,6 +20,35 @@ # Please add new experiment requests towards the top of this file. # +- experiment: 2022-10-07-um-full + description: "UM fuzzer experiment" + fuzzers: + - aflplusplus + - aflplusplus_um_parallel + - aflplusplus + - libfuzzer_um_prioritize + - libfuzzer_um_random + - libfuzzer_um_parallel + - libfuzzer + - afl_um_prioritize + - afl_um_random + - afl_um_parallel + - afl + - eclipser_um_prioritize + - eclipser_um_random + - eclipser_um_parallel + - eclipser + - honggfuzz_um_prioritize + - honggfuzz_um_random + - honggfuzz_um_parallel + - honggfuzz + +- experiment: 2022-10-07-um-3 + description: "UM fuzzer experiment" + fuzzers: + - aflplusplus + - aflplusplus_um_random_3 + - experiment: 2022-10-06-um-full description: "UM fuzzer experiment" fuzzers: