Merge remote-tracking branch 'origin/google_master' into seed_exp

.github/workflows/benchmarks.yml

@@ -33,6 +33,7 @@ jobs:
           - libxslt_xpath
           - mbedtls_fuzz_dtlsclient
           - mbedtls_fuzz_dtlsclient_7c6b0e
+          - mruby_mruby_fuzzer_8c8bbd
           - openh264_decoder_fuzzer
           - openssl_x509
           - openthread_ot-ip6-send-fuzzer

.gitignore

@@ -44,3 +44,6 @@ docker/generated.mk
 
 # Vim backup files.
 .*.swp
+
+# Diff files from matplotlib
+*-failed-diff.png

benchmarks/mruby_mruby_fuzzer_8c8bbd/Dockerfile

@@ -0,0 +1,24 @@
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM gcr.io/oss-fuzz-base/base-builder@sha256:87ca1e9e19235e731fac8de8d1892ebe8d55caf18e7aa131346fc582a2034fdd
+RUN apt-get update && apt-get install -y build-essential ruby bison ninja-build \
+    cmake zlib1g-dev libbz2-dev liblzma-dev
+RUN git clone \
+        https://github.com/mruby/mruby
+RUN git clone --depth 1 https://github.com/bshastry/mruby_seeds.git mruby_seeds
+WORKDIR mruby
+COPY build.sh *.c *.options *.dict $SRC/

benchmarks/mruby_mruby_fuzzer_8c8bbd/benchmark.yaml

@@ -0,0 +1,6 @@
+commit: 8c8bbd94dce3b3eabcf72c674e690516c075b0ee
+commit_date: 2023-02-03T04:41:10+0000
+fuzz_target: mruby_fuzzer
+project: mruby
+type: bug
+unsupported_fuzzers:

benchmarks/mruby_mruby_fuzzer_8c8bbd/build.sh

@@ -0,0 +1,44 @@
+#!/bin/bash -eu
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# Instrument mruby
+(
+cd $SRC/mruby
+export LD=$CC
+export LDFLAGS="$CFLAGS -fPIE"
+
+# No you cannot overwrite CC/CXX, otherwise afl++, libafl, etc. wont work!
+#LD=/usr/local/bin/clang CC=/usr/local/bin/clang CXX=/usr/local/bin/clang++ rake -m || true
+rake -m || true
+
+test -f $SRC/mruby/build/host/lib/libmruby.a
+
+# build fuzzers
+FUZZ_TARGET=$SRC/mruby_fuzzer.c
+name=$(basename $FUZZ_TARGET .c)
+$CC -c $CFLAGS -Iinclude \
+     ${FUZZ_TARGET} -o $OUT/${name}.o
+$CXX $CXXFLAGS $OUT/${name}.o $LIB_FUZZING_ENGINE -lm \
+    $SRC/mruby/build/host/lib/libmruby.a -o $OUT/${name}
+rm -f $OUT/${name}.o
+)
+
+# dict
+cp $SRC/mruby.dict $OUT/mruby_fuzzer.dict
+
+# seeds
+zip -rq $OUT/mruby_fuzzer_seed_corpus $SRC/mruby_seeds

benchmarks/mruby_mruby_fuzzer_8c8bbd/mruby.dict

@@ -0,0 +1,105 @@
+keyword___ENCODING__="__ENCODING__"
+keyword___FILE__="__FILE__"
+keyword___LINE__="__LINE__"
+keyword_BEGIN="BEGIN"
+keyword_END="END"
+keyword_alias="alias"
+keyword_and="and"
+keyword_begin="begin"
+keyword_break="break"
+keyword_case="case"
+keyword_class="class"
+keyword_def="def"
+keyword_do="do"
+keyword_else="else"
+keyword_elsif="elsif"
+keyword_end="end"
+keyword_ensure="ensure"
+keyword_false="false"
+keyword_for="for"
+keyword_if="if"
+keyword_in="in"
+keyword_module="module"
+keyword_next="next"
+keyword_nil="nil"
+keyword_not="not"
+keyword_or="or"
+keyword_redo="redo"
+keyword_rescue="rescue"
+keyword_retry="retry"
+keyword_return="return"
+keyword_self="self"
+keyword_super="super"
+keyword_then="then"
+keyword_true="true"
+keyword_undef="undef"
+keyword_unless="unless"
+keyword_until="until"
+keyword_when="when"
+keyword_while="while"
+keyword_yield="yield"
+
+operator_a=" !"
+operator_b=" ~"
+operator_c=" +"
+operator_d=" -"
+operator_e=" []"
+operator_f=" []="
+operator_g=" *"
+operator_h=" /"
+operator_i=" %"
+operator_j=" +-"
+operator_k=" >>"
+operator_l=" <<"
+operator_m=" &"
+operator_n=" ^"
+operator_o=" |"
+operator_p=" <="
+operator_q=" <>"
+operator_r=" >="
+operator_s=" <=>"
+operator_t=" =="
+operator_u=" ==="
+operator_v=" !="
+operator_w=" =~"
+operator_x=" !~"
+operator_y=" &&"
+operator_z=" ||"
+operator_aa=" .."
+operator_ab=" ..."
+operator_ac=" ?"
+operator_ad=" :"
+operator_ae=" ="
+operator_af=" %="
+operator_ag=" /="
+operator_ah=" -="
+operator_ai=" +="
+operator_aj=" |="
+operator_ak=" &="
+operator_al=" >>="
+operator_am=" <<="
+operator_an=" *="
+operator_ao=" &&="
+operator_ap=" ||="
+operator_aq=" **="
+operator_ar=" ^="
+operator_as=" not"
+operator_at=" or"
+operator_au=" and"
+operator_av=" if"
+operator_aw=" unless"
+operator_ax=" while"
+operator_ay=" until"
+operator_az=" begin"
+operator_ba=" end"
+
+snippet_1eq1=" 1=1"
+snippet_dollar=" $1"
+snippet_at=" @a"
+snippet_symbol=" :a"
+snippet_array=" [1,2]"
+snippet_block=" 1.times{|x| x}"
+snippet_multi=" 1*1"
+
+string_single_q=" 'a'"
+string_dbl_q=" \"a\""

benchmarks/mruby_mruby_fuzzer_8c8bbd/mruby_fuzzer.c

@@ -0,0 +1,18 @@
+#include <stdlib.h>
+#include <string.h>
+#include <mruby.h>
+#include <mruby/compile.h>
+
+int LLVMFuzzerTestOneInput(uint8_t *Data, size_t size) {
+    if (size < 1) {
+        return 0;
+    }
+    char *code = malloc(size+1);
+    memcpy(code, Data, size);
+    code[size] = '\0';
+    mrb_state *mrb = mrb_open();
+    mrb_load_string(mrb, code);
+    mrb_close(mrb);
+    free(code);
+    return 0;
+}

common/experiment_utils.py

@@ -129,6 +129,11 @@ def get_corpus_archive_name(cycle: int) -> str:
     return get_cycle_filename('corpus-archive', cycle) + '.tar.gz'
 
 
+def get_coverage_archive_name(cycle: int) -> str:
+    """Returns a corpus archive name given a cycle."""
+    return get_cycle_filename('coverage-archive', cycle) + '.json'
+
+
 def get_stats_filename(cycle: int) -> str:
     """Returns a corpus archive name given a cycle."""
     return get_cycle_filename('stats', cycle) + '.json'

experiment/measurer/measure_manager.py

@@ -16,6 +16,7 @@
 import collections
 import gc
 import glob
+import gzip
 import multiprocessing
 import json
 import os
@@ -614,10 +615,33 @@ def measure_snapshot_coverage(  # pylint: disable=too-many-locals
     # Generate profdata and transform it into json form.
     snapshot_measurer.generate_coverage_information(cycle)
 
+    # Compress and save the exported profdata snapshot.
+    coverage_archive_zipped = os.path.join(
+        snapshot_measurer.trial_dir, 'coverage',
+        experiment_utils.get_coverage_archive_name(cycle) + '.gz')
+
+    coverage_archive_dir = os.path.dirname(coverage_archive_zipped)
+    if not os.path.exists(coverage_archive_dir):
+        os.makedirs(coverage_archive_dir)
+
+    with gzip.open(str(coverage_archive_zipped), 'wb') as compressed:
+        with open(snapshot_measurer.cov_summary_file, 'rb') as uncompressed:
+            # avoid saving warnings so we can direct import with pandas
+            compressed.write(uncompressed.readlines()[-1])
+
+    coverage_archive_dst = exp_path.filestore(coverage_archive_zipped)
+    if filestore_utils.cp(coverage_archive_zipped,
+                          coverage_archive_dst,
+                          expect_zero=False).retcode:
+        snapshot_logger.warning('Coverage not found for cycle: %d.', cycle)
+        return None
+
+    os.remove(coverage_archive_zipped)  # no reason to keep this around
+
     # Run crashes again, parse stacktraces and generate crash signatures.
     crashes = snapshot_measurer.process_crashes(cycle)
 
-    # Get the coverage of the new corpus units.
+    # Get the coverage summary of the new corpus units.
     branches_covered = snapshot_measurer.get_current_coverage()
     fuzzer_stats_data = snapshot_measurer.get_fuzzer_stats(cycle)
     snapshot = models.Snapshot(time=this_time,

fuzzers/gramatron/builder.Dockerfile

@@ -17,7 +17,7 @@ FROM $parent_image
 
 # Install libstdc++ to use llvm_mode.
 RUN apt-get update && \
-    apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \
+    apt-get install -y wget libstdc++-10-dev libtool-bin automake flex bison \
                        libglib2.0-dev libpixman-1-dev python3-setuptools unzip \
                        apt-utils apt-transport-https ca-certificates joe curl \
                        python3-dev gzip
@@ -32,14 +32,14 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /rustup.sh && \
 # Download libafl
 RUN git clone https://github.com/AFLplusplus/libafl_fuzzbench /libafl_fuzzbench && \
     cd /libafl_fuzzbench && \
-    git checkout db32b7b8c1c0065a0cec2129b4dfe3897d1b9a4b && \
+    git checkout b7fc9fd143daff0190fd623ed3a8b9fbc64cc00c && \
     git submodule update --init
 
 # Compile libafl
 RUN cd /libafl_fuzzbench/ && unset CFLAGS && unset CXXFLAGS && \
     export CC=clang && export CXX=clang++ && \
     export LIBAFL_EDGES_MAP_SIZE=2621440 && \
-    PATH="$PATH:/root/.cargo/bin/" cargo build --release
+    PATH="/root/.cargo/bin:$PATH" cargo build --release -p gramatron
 
 RUN wget https://gist.githubusercontent.com/andreafioraldi/e5f60d68c98b31665a274207cfd05541/raw/4da351a321f1408df566a9cf2ce7cde6eeab3904/empty_fuzzer_lib.c -O /empty_fuzzer_lib.c && \
     clang -c /empty_fuzzer_lib.c && \

fuzzers/gramatron/fuzzer.yaml

@@ -1,4 +1,4 @@
 allowed_benchmarks:
   - quickjs_eval-2020-01-05
   - php_php-fuzz-execute
-  - mruby-2018-05-23
+  - mruby_mruby_fuzzer_8c8bbd

fuzzers/grimoire/builder.Dockerfile

@@ -17,7 +17,7 @@ FROM $parent_image
 
 # Install libstdc++ to use llvm_mode.
 RUN apt-get update && \
-    apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \
+    apt-get install -y wget libstdc++-10-dev libtool-bin automake flex bison \
                        libglib2.0-dev libpixman-1-dev python3-setuptools unzip \
                        apt-utils apt-transport-https ca-certificates joe curl \
                        python3-dev gzip
@@ -32,14 +32,14 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /rustup.sh && \
 # Download libafl
 RUN git clone https://github.com/AFLplusplus/libafl_fuzzbench /libafl_fuzzbench && \
     cd /libafl_fuzzbench && \
-    git checkout db32b7b8c1c0065a0cec2129b4dfe3897d1b9a4b && \
+    git checkout b7fc9fd143daff0190fd623ed3a8b9fbc64cc00c && \
     git submodule update --init
 
 # Compile libafl
 RUN cd /libafl_fuzzbench/ && unset CFLAGS && unset CXXFLAGS && \
     export CC=clang && export CXX=clang++ && \
     export LIBAFL_EDGES_MAP_SIZE=2621440 && \
-    PATH="$PATH:/root/.cargo/bin/" cargo build --release
+    PATH="/root/.cargo/bin:$PATH" cargo build --release -p grimoire
 
 RUN wget https://gist.githubusercontent.com/andreafioraldi/e5f60d68c98b31665a274207cfd05541/raw/4da351a321f1408df566a9cf2ce7cde6eeab3904/empty_fuzzer_lib.c -O /empty_fuzzer_lib.c && \
     clang -c /empty_fuzzer_lib.c && \

fuzzers/grimoire/fuzzer.yaml

@@ -1,4 +1,4 @@
 allowed_benchmarks:
   - quickjs_eval-2020-01-05
   - php_php-fuzz-execute
-  - mruby-2018-05-23
+  - mruby_mruby_fuzzer_8c8bbd

fuzzers/nautilus/builder.Dockerfile

@@ -17,7 +17,7 @@ FROM $parent_image
 
 # Install libstdc++ to use llvm_mode.
 RUN apt-get update && \
-    apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \
+    apt-get install -y wget libstdc++-10-dev libtool-bin automake flex bison \
                        libglib2.0-dev libpixman-1-dev python3-setuptools unzip \
                        apt-utils apt-transport-https ca-certificates joe curl \
                        python3-dev gzip
@@ -32,14 +32,14 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /rustup.sh && \
 # Download libafl
 RUN git clone https://github.com/AFLplusplus/libafl_fuzzbench /libafl_fuzzbench && \
     cd /libafl_fuzzbench && \
-    git checkout db32b7b8c1c0065a0cec2129b4dfe3897d1b9a4b && \
+    git checkout b7fc9fd143daff0190fd623ed3a8b9fbc64cc00c && \
     git submodule update --init
 
 # Compile libafl
 RUN cd /libafl_fuzzbench/ && unset CFLAGS && unset CXXFLAGS && \
     export CC=clang && export CXX=clang++ && \
     export LIBAFL_EDGES_MAP_SIZE=2621440 && \
-    PATH="$PATH:/root/.cargo/bin/" cargo build --release
+    PATH="/root/.cargo/bin:$PATH" cargo build --release -p nautilus
 
 RUN wget https://gist.githubusercontent.com/andreafioraldi/e5f60d68c98b31665a274207cfd05541/raw/4da351a321f1408df566a9cf2ce7cde6eeab3904/empty_fuzzer_lib.c -O /empty_fuzzer_lib.c && \
     clang -c /empty_fuzzer_lib.c && \

fuzzers/nautilus/fuzzer.yaml

@@ -1,4 +1,4 @@
 allowed_benchmarks:
   - quickjs_eval-2020-01-05
   - php_php-fuzz-execute
-  - mruby-2018-05-23
+  - mruby_mruby_fuzzer_8c8bbd

fuzzers/nautilus/runner.Dockerfile

@@ -14,6 +14,8 @@
 
 FROM gcr.io/fuzzbench/base-image
 
+RUN apt update && apt -y install libexpat1-dev zlib1g-dev
+
 # This makes interactive docker runs painless:
 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out"
 #ENV AFL_MAP_SIZE=2621440