From 52374a00e9ca1cc7ddb54063a8f83fee89b05d19 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 14 Aug 2024 11:04:27 +0200 Subject: [PATCH] Add mutant-based benchmarks and update experiments data in YAML file host seeds elsewhere modified files needed to run the experiment fixed date --- .../Dockerfile | 37 +++++++++++ .../benchmark.yaml | 24 +++++++ .../build.sh | 23 +++++++ .../cms_transform_fuzzer.cc | 61 ++++++++++++++++++ .../Dockerfile | 37 +++++++++++ .../benchmark.yaml | 24 +++++++ .../build.sh | 23 +++++++ .../cms_transform_fuzzer.cc | 61 ++++++++++++++++++ .../Dockerfile | 37 +++++++++++ .../benchmark.yaml | 24 +++++++ .../build.sh | 23 +++++++ .../cms_transform_fuzzer.cc | 61 ++++++++++++++++++ service/experiment-requests.yaml | 64 ++++++++++--------- service/gcbrun_experiment.py | 2 +- 14 files changed, 471 insertions(+), 30 deletions(-) create mode 100644 benchmarks/lcms_cms_transform_fuzzer_all_seeds/Dockerfile create mode 100644 benchmarks/lcms_cms_transform_fuzzer_all_seeds/benchmark.yaml create mode 100755 benchmarks/lcms_cms_transform_fuzzer_all_seeds/build.sh create mode 100644 benchmarks/lcms_cms_transform_fuzzer_all_seeds/cms_transform_fuzzer.cc create mode 100644 benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/Dockerfile create mode 100644 benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/benchmark.yaml create mode 100755 benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/build.sh create mode 100644 benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/cms_transform_fuzzer.cc create mode 100644 benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/Dockerfile create mode 100644 benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/benchmark.yaml create mode 100755 benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/build.sh create mode 100644 benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/cms_transform_fuzzer.cc diff --git a/benchmarks/lcms_cms_transform_fuzzer_all_seeds/Dockerfile b/benchmarks/lcms_cms_transform_fuzzer_all_seeds/Dockerfile new file mode 100644 index 000000000..8760ca351 --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_all_seeds/Dockerfile @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +FROM gcr.io/oss-fuzz-base/base-builder@sha256:87ca1e9e19235e731fac8de8d1892ebe8d55caf18e7aa131346fc582a2034fdd + +RUN apt-get update && \ + apt-get install -y \ + make \ + automake \ + libtool \ + wget + +RUN git clone https://github.com/mm2/Little-CMS.git + +RUN wget -qO $OUT/cms_transform_fuzzer.dict \ + https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/icc.dict +COPY cms_transform_fuzzer.cc build.sh $SRC/ +# Download the seeds tarball from your GitHub repository +RUN wget https://raw.githubusercontent.com/ardier/fuzzbench/minimized-subsumed-mutants-benchmark-with-seeds/benchmarks/lcms_cms_transform_fuzzer_all_seeds/seeds.tar.gz -O /tmp/seeds.tar.gz + +# Extract the seeds tarball to the /opt/seeds directory +RUN mkdir -p /opt/seeds && \ + tar -xzvf /tmp/seeds.tar.gz -C /opt/seeds +# ADD seeds /opt/seeds diff --git a/benchmarks/lcms_cms_transform_fuzzer_all_seeds/benchmark.yaml b/benchmarks/lcms_cms_transform_fuzzer_all_seeds/benchmark.yaml new file mode 100644 index 000000000..53204c859 --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_all_seeds/benchmark.yaml @@ -0,0 +1,24 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +commit: f0d963261b28253999e239a844ac74d5a8960f40 +commit_date: 2023-01-25T18:20:28+0000 +fuzz_target: cms_transform_fuzzer +project: lcms +unsupported_fuzzers: + - symcc_afl + - symcc_afl_single + - symcc_aflplusplus + - afldd + - aflpp_vs_dd diff --git a/benchmarks/lcms_cms_transform_fuzzer_all_seeds/build.sh b/benchmarks/lcms_cms_transform_fuzzer_all_seeds/build.sh new file mode 100755 index 000000000..3e559e072 --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_all_seeds/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash -ex +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd Little-CMS +./autogen.sh +./configure +make -j $(nproc) + +$CXX $CXXFLAGS $SRC/cms_transform_fuzzer.cc -I include/ src/.libs/liblcms2.a \ + $FUZZER_LIB -o $OUT/cms_transform_fuzzer +cp -r /opt/seeds $OUT/ diff --git a/benchmarks/lcms_cms_transform_fuzzer_all_seeds/cms_transform_fuzzer.cc b/benchmarks/lcms_cms_transform_fuzzer_all_seeds/cms_transform_fuzzer.cc new file mode 100644 index 000000000..e86a9bf5c --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_all_seeds/cms_transform_fuzzer.cc @@ -0,0 +1,61 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "lcms2.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + cmsHPROFILE srcProfile = cmsOpenProfileFromMem(data, size); + if (!srcProfile) return 0; + + cmsHPROFILE dstProfile = cmsCreate_sRGBProfile(); + if (!dstProfile) { + cmsCloseProfile(srcProfile); + return 0; + } + + cmsColorSpaceSignature srcCS = cmsGetColorSpace(srcProfile); + cmsUInt32Number nSrcComponents = cmsChannelsOf(srcCS); + cmsUInt32Number srcFormat; + if (srcCS == cmsSigLabData) { + srcFormat = + COLORSPACE_SH(PT_Lab) | CHANNELS_SH(nSrcComponents) | BYTES_SH(0); + } else { + srcFormat = + COLORSPACE_SH(PT_ANY) | CHANNELS_SH(nSrcComponents) | BYTES_SH(1); + } + + cmsUInt32Number intent = 0; + cmsUInt32Number flags = 0; + cmsHTRANSFORM hTransform = cmsCreateTransform( + srcProfile, srcFormat, dstProfile, TYPE_BGR_8, intent, flags); + cmsCloseProfile(srcProfile); + cmsCloseProfile(dstProfile); + if (!hTransform) return 0; + + uint8_t output[4]; + if (T_BYTES(srcFormat) == 0) { // 0 means double + double input[nSrcComponents]; + for (uint32_t i = 0; i < nSrcComponents; i++) input[i] = 0.5f; + cmsDoTransform(hTransform, input, output, 1); + } else { + uint8_t input[nSrcComponents]; + for (uint32_t i = 0; i < nSrcComponents; i++) input[i] = 128; + cmsDoTransform(hTransform, input, output, 1); + } + cmsDeleteTransform(hTransform); + + return 0; +} diff --git a/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/Dockerfile b/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/Dockerfile new file mode 100644 index 000000000..54941326f --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/Dockerfile @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +FROM gcr.io/oss-fuzz-base/base-builder@sha256:87ca1e9e19235e731fac8de8d1892ebe8d55caf18e7aa131346fc582a2034fdd + +RUN apt-get update && \ + apt-get install -y \ + make \ + automake \ + libtool \ + wget + +RUN git clone https://github.com/mm2/Little-CMS.git + +RUN wget -qO $OUT/cms_transform_fuzzer.dict \ + https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/icc.dict +COPY cms_transform_fuzzer.cc build.sh $SRC/ +# Download the seeds tarball from your GitHub repository +RUN wget https://raw.githubusercontent.com/ardier/fuzzbench/minimized-subsumed-mutants-benchmark-with-seeds/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/seeds.tar.gz -O /tmp/seeds.tar.gz + +# Extract the seeds tarball to the /opt/seeds directory +RUN mkdir -p /opt/seeds && \ + tar -xzvf /tmp/seeds.tar.gz +# ADD seeds /opt/seeds diff --git a/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/benchmark.yaml b/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/benchmark.yaml new file mode 100644 index 000000000..53204c859 --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/benchmark.yaml @@ -0,0 +1,24 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +commit: f0d963261b28253999e239a844ac74d5a8960f40 +commit_date: 2023-01-25T18:20:28+0000 +fuzz_target: cms_transform_fuzzer +project: lcms +unsupported_fuzzers: + - symcc_afl + - symcc_afl_single + - symcc_aflplusplus + - afldd + - aflpp_vs_dd diff --git a/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/build.sh b/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/build.sh new file mode 100755 index 000000000..3e559e072 --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash -ex +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd Little-CMS +./autogen.sh +./configure +make -j $(nproc) + +$CXX $CXXFLAGS $SRC/cms_transform_fuzzer.cc -I include/ src/.libs/liblcms2.a \ + $FUZZER_LIB -o $OUT/cms_transform_fuzzer +cp -r /opt/seeds $OUT/ diff --git a/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/cms_transform_fuzzer.cc b/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/cms_transform_fuzzer.cc new file mode 100644 index 000000000..e86a9bf5c --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_dominator_mutants/cms_transform_fuzzer.cc @@ -0,0 +1,61 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "lcms2.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + cmsHPROFILE srcProfile = cmsOpenProfileFromMem(data, size); + if (!srcProfile) return 0; + + cmsHPROFILE dstProfile = cmsCreate_sRGBProfile(); + if (!dstProfile) { + cmsCloseProfile(srcProfile); + return 0; + } + + cmsColorSpaceSignature srcCS = cmsGetColorSpace(srcProfile); + cmsUInt32Number nSrcComponents = cmsChannelsOf(srcCS); + cmsUInt32Number srcFormat; + if (srcCS == cmsSigLabData) { + srcFormat = + COLORSPACE_SH(PT_Lab) | CHANNELS_SH(nSrcComponents) | BYTES_SH(0); + } else { + srcFormat = + COLORSPACE_SH(PT_ANY) | CHANNELS_SH(nSrcComponents) | BYTES_SH(1); + } + + cmsUInt32Number intent = 0; + cmsUInt32Number flags = 0; + cmsHTRANSFORM hTransform = cmsCreateTransform( + srcProfile, srcFormat, dstProfile, TYPE_BGR_8, intent, flags); + cmsCloseProfile(srcProfile); + cmsCloseProfile(dstProfile); + if (!hTransform) return 0; + + uint8_t output[4]; + if (T_BYTES(srcFormat) == 0) { // 0 means double + double input[nSrcComponents]; + for (uint32_t i = 0; i < nSrcComponents; i++) input[i] = 0.5f; + cmsDoTransform(hTransform, input, output, 1); + } else { + uint8_t input[nSrcComponents]; + for (uint32_t i = 0; i < nSrcComponents; i++) input[i] = 128; + cmsDoTransform(hTransform, input, output, 1); + } + cmsDeleteTransform(hTransform); + + return 0; +} diff --git a/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/Dockerfile b/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/Dockerfile new file mode 100644 index 000000000..fe6b3fe07 --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/Dockerfile @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +FROM gcr.io/oss-fuzz-base/base-builder@sha256:87ca1e9e19235e731fac8de8d1892ebe8d55caf18e7aa131346fc582a2034fdd + +RUN apt-get update && \ + apt-get install -y \ + make \ + automake \ + libtool \ + wget + +RUN git clone https://github.com/mm2/Little-CMS.git + +RUN wget -qO $OUT/cms_transform_fuzzer.dict \ + https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/icc.dict +COPY cms_transform_fuzzer.cc build.sh $SRC/ +# Download the seeds tarball from your GitHub repository +RUN wget https://raw.githubusercontent.com/ardier/fuzzbench/minimized-subsumed-mutants-benchmark-with-seeds/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/seeds.tar.gz -O /tmp/seeds.tar.gz + +# Extract the seeds tarball to the /opt/seeds directory +RUN mkdir -p /opt/seeds && \ + tar -xzvf /tmp/seeds.tar.gz +# ADD seeds /opt/seeds diff --git a/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/benchmark.yaml b/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/benchmark.yaml new file mode 100644 index 000000000..53204c859 --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/benchmark.yaml @@ -0,0 +1,24 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +commit: f0d963261b28253999e239a844ac74d5a8960f40 +commit_date: 2023-01-25T18:20:28+0000 +fuzz_target: cms_transform_fuzzer +project: lcms +unsupported_fuzzers: + - symcc_afl + - symcc_afl_single + - symcc_aflplusplus + - afldd + - aflpp_vs_dd diff --git a/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/build.sh b/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/build.sh new file mode 100755 index 000000000..3e559e072 --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash -ex +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd Little-CMS +./autogen.sh +./configure +make -j $(nproc) + +$CXX $CXXFLAGS $SRC/cms_transform_fuzzer.cc -I include/ src/.libs/liblcms2.a \ + $FUZZER_LIB -o $OUT/cms_transform_fuzzer +cp -r /opt/seeds $OUT/ diff --git a/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/cms_transform_fuzzer.cc b/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/cms_transform_fuzzer.cc new file mode 100644 index 000000000..e86a9bf5c --- /dev/null +++ b/benchmarks/lcms_cms_transform_fuzzer_minimized_mutants/cms_transform_fuzzer.cc @@ -0,0 +1,61 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "lcms2.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + cmsHPROFILE srcProfile = cmsOpenProfileFromMem(data, size); + if (!srcProfile) return 0; + + cmsHPROFILE dstProfile = cmsCreate_sRGBProfile(); + if (!dstProfile) { + cmsCloseProfile(srcProfile); + return 0; + } + + cmsColorSpaceSignature srcCS = cmsGetColorSpace(srcProfile); + cmsUInt32Number nSrcComponents = cmsChannelsOf(srcCS); + cmsUInt32Number srcFormat; + if (srcCS == cmsSigLabData) { + srcFormat = + COLORSPACE_SH(PT_Lab) | CHANNELS_SH(nSrcComponents) | BYTES_SH(0); + } else { + srcFormat = + COLORSPACE_SH(PT_ANY) | CHANNELS_SH(nSrcComponents) | BYTES_SH(1); + } + + cmsUInt32Number intent = 0; + cmsUInt32Number flags = 0; + cmsHTRANSFORM hTransform = cmsCreateTransform( + srcProfile, srcFormat, dstProfile, TYPE_BGR_8, intent, flags); + cmsCloseProfile(srcProfile); + cmsCloseProfile(dstProfile); + if (!hTransform) return 0; + + uint8_t output[4]; + if (T_BYTES(srcFormat) == 0) { // 0 means double + double input[nSrcComponents]; + for (uint32_t i = 0; i < nSrcComponents; i++) input[i] = 0.5f; + cmsDoTransform(hTransform, input, output, 1); + } else { + uint8_t input[nSrcComponents]; + for (uint32_t i = 0; i < nSrcComponents; i++) input[i] = 128; + cmsDoTransform(hTransform, input, output, 1); + } + cmsDeleteTransform(hTransform); + + return 0; +} diff --git a/service/experiment-requests.yaml b/service/experiment-requests.yaml index fe0efe2a2..c42cbe470 100644 --- a/service/experiment-requests.yaml +++ b/service/experiment-requests.yaml @@ -20,6 +20,17 @@ # Please add new experiment requests towards the top of this file. # +- experiment: 2024-09-03-afl-mutants + description: "Benchmark afl and afl++ with minizmied mutants" + fuzzers: + - afl + - aflplusplus + benchmarks: + - lcms_cms_transform_fuzzer + - lcms_cms_transform_fuzzer_all_seeds + - lcms_cms_transform_fuzzer_minimized_mutants + - lcms_cms_transform_fuzzer_dominator_mutants + - experiment: 2023-06-12-aflpp description: "Benchmark afl++ releases and newmutation" fuzzers: @@ -1027,13 +1038,13 @@ - harfbuzz-1.3.2 - lcms-2017-03-21 - libjpeg-turbo-07-2017 - - mbedtls_fuzz_dtlsclient + - mbedtls_fuzz_dtlsclient - openthread-2019-12-23 - proj4-2017-08-14 - re2-2014-12-09 - sqlite3_ossfuzz - vorbis-2017-12-11 - - zlib_zlib_uncompress_fuzzer + - zlib_zlib_uncompress_fuzzer - experiment: 2022-10-07-um-3b description: "UM fuzzer experiment" @@ -1135,7 +1146,7 @@ fuzzers: - aflplusplus - aflplusplus_um_random - + - experiment: 2022-09-29-wingfuzz description: "Wingfuzz coverage experiment (compare against core fuzzers)" fuzzers: @@ -1151,7 +1162,7 @@ - libfuzzer - mopt - wingfuzz - + - experiment: 2022-09-29 description: "Main coverage experiment" type: code @@ -1598,7 +1609,6 @@ - aflfast - fairfuzz - - experiment: 2022-04-10-aflpp description: "afl++ bug ranking" type: bug @@ -2494,20 +2504,20 @@ - experiment: 2021-06-02-symccafl-pp description: > - Symcc experiment. The difference between this and - 2021-06-01-symccafl is that we now have a version of - symcc in combination with aflplusplus. Some bug fixing in - symcc has happened, which makes it less prone to crashing. - The bug fixing is primarily for the aflplusplus hybrid, - since the bugs that were fixed has not been notable - (maybe seen once) in the symcc-afl combination. - Finally, we have added an increase in the timeout - of how often symcc runs, switching it from every - 5 sec to ever 20 sec. Finally, in the aflplusplus - hybrid there is no use of afl-showmap, which means - all seeds created by symcc are pushed into the afl - queue. This changes the symcc set up as there is no - filtering done on the seeds pushed to afl. + Symcc experiment. The difference between this and + 2021-06-01-symccafl is that we now have a version of + symcc in combination with aflplusplus. Some bug fixing in + symcc has happened, which makes it less prone to crashing. + The bug fixing is primarily for the aflplusplus hybrid, + since the bugs that were fixed has not been notable + (maybe seen once) in the symcc-afl combination. + Finally, we have added an increase in the timeout + of how often symcc runs, switching it from every + 5 sec to ever 20 sec. Finally, in the aflplusplus + hybrid there is no use of afl-showmap, which means + all seeds created by symcc are pushed into the afl + queue. This changes the symcc set up as there is no + filtering done on the seeds pushed to afl. fuzzers: - symcc_afl - symcc_afl_single @@ -2527,12 +2537,12 @@ - experiment: 2021-06-01-symccafl description: > - Symcc-AFL test. The difference between this and - 2021-05-29-symccafl is that we now have afl benchmarks - that run multiple instances of afl. This is for - comparison purposes since symcc also utilises mutliple - processes. The goal is to try and avoid any bias due - to more total CPU time. + Symcc-AFL test. The difference between this and + 2021-05-29-symccafl is that we now have afl benchmarks + that run multiple instances of afl. This is for + comparison purposes since symcc also utilises mutliple + processes. The goal is to try and avoid any bias due + to more total CPU time. fuzzers: - symcc_afl - symcc_afl_single @@ -3322,7 +3332,6 @@ - libfuzzer - mopt - - experiment: 2020-10-26 description: "test new memcache algorithm, hopefully for the last time" fuzzers: @@ -3331,7 +3340,6 @@ - aflplusplus_memcache_20mb - aflplusplus_memcache_200mb - - experiment: 2020-10-25 description: > These are AFLPlusplus experiments. Using Marc's implementation of the Vose @@ -3356,7 +3364,6 @@ - aflplusplus_fast_branches_v2 - aflplusplus - - experiment: 2020-10-24 description: > There are mainly two versions; one boosts via perf_score and one that @@ -3383,7 +3390,6 @@ - afl_fast - afl - - experiment: 2020-10-23-2 description: > Experiment aflplusplus_cmplog and aflplusplus_dict2file variants with other fuzzers diff --git a/service/gcbrun_experiment.py b/service/gcbrun_experiment.py index f19ab493d..2652042b3 100644 --- a/service/gcbrun_experiment.py +++ b/service/gcbrun_experiment.py @@ -16,7 +16,7 @@ """Entrypoint for gcbrun into run_experiment. This script will get the command from the last PR comment containing "/gcbrun" and pass it to run_experiment.py which will run an experiment.""" - +# a dummy comment for experiment! import logging import os import sys