Skip to content

Commit

Permalink
interim
Browse files Browse the repository at this point in the history
  • Loading branch information
@wintered
wintered committed May 31, 2021
1 parent ee147ac commit bb19017
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 40 deletions.
61 changes: 43 additions & 18 deletions src/modules/Fuzzer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,17 @@
from src.generators.TypeAwareOpMutation import TypeAwareOpMutation
from src.generators.SemanticFusion.SemanticFusion import SemanticFusion

class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKCYAN = '\033[96m'
OKGREEN = '\033[92m'
WARNING = '\033[91m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'

class Fuzzer:

def __init__(self, args):
Expand All @@ -31,6 +42,7 @@ def __init__(self, args):
self.generator = None
self.old_time = time.time()
self.start_time = time.time()
self.first_status_bar_printed = False

# Init logging
log_fn = self.args.logfolder+"/"+str(self.args.name)
Expand Down Expand Up @@ -72,18 +84,19 @@ def run(self):
seed = seeds.pop(random.randrange(len(seeds)))

logging.debug("Processing seed "+seed)
self.statistic.total_seeds += 1

self.statistic.seeds += 1
if not self.admissible_seed_size(seed):
self.statistic.invalid_seeds += 1

logging.debug("Skipping invalid seed: exceeds max file size")
self.statistic.ignored += 1
continue

self.currentseeds = Path(seed).stem
script = parse_file(seed,silent=True)

if not script: # i.e. parsing was unsucessful
self.statistic.ignored += 1
self.statistic.invalid_seeds += 1
logging.debug("Skipping invalid seed: error in parsing")
continue

Expand Down Expand Up @@ -120,21 +133,30 @@ def run(self):
unsuccessful=0
for i in range(self.args.iterations):
if not self.args.quiet:
if not self.first_status_bar_printed and time.time() - self.old_time >= 1:
self.statistic.printbar(self.start_time)
self.old_time = time.time()
self.first_status_bar_printed = True


if time.time() - self.old_time >= 5.0:
self.statistic.printbar(self.start_time)
self.old_time = time.time()

formula, success, skip_seed = self.generator.generate()
if not success:
self.statistic.unsuccessful_generations += 1
unsuccessful+=1
continue

if not self.test(formula,i): break
if not self.test(formula,i+1): break
self.statistic.mutants += 1
if skip_seed: break

successful= self.args.iterations - unsuccessful
successful = self.args.iterations - unsuccessful
logging.debug("Finished generations: "+ str(successful)+" successful, "+ str(unsuccessful)+ " unsuccessful")
print("All seeds processed")
self.statistic.printsum()

def create_testbook(self, formula):
testbook = []
Expand Down Expand Up @@ -192,7 +214,7 @@ def init_oracle(self):
assert(False)


def test(self, formula,i):
def test(self, formula, i):
"""
Tests the solvers with the formula returning "False" if the testing on
formula should be stopped and "True" otherwise.
Expand All @@ -212,10 +234,11 @@ def test(self, formula,i):

# (2) Match against the duplicate list to avoid reporting duplicate bugs.
if not self.in_duplicate_list(stdout, stderr):
self.statistic.effective_call += 1
self.statistic.effective_calls += 1
self.statistic.crashes += 1
self.report(scratchfile, "crash", solver_cli, stdout, stderr, random_string())
_,path = self.report(scratchfile, "crash", solver_cli, stdout, stderr, random_string())
logging.debug("Crash! Stop testing on this seed.")
logging.info(bcolors.BOLD+bcolors.WARNING+"Detected crash bug: "+ path+bcolors.ENDC)
else:
self.statistic.duplicates += 1
logging.debug("Duplicate. Stop testing on this seed.")
Expand All @@ -225,16 +248,17 @@ def test(self, formula,i):
# the ignore list.
if self.in_ignore_list(stdout, stderr):
logging.debug("Invalid mutant:ignore_list. solver="+str(solver_cli))
self.statistic.ignored += 1
self.statistic.invalid_mutants+= 1
continue # continue with next solver (4)

# (3b) Check whether the exit code is nonzero.
if exitcode != 0:
if exitcode == -signal.SIGSEGV or exitcode == 245: #segfault
self.statistic.effective_call += 1
self.statistic.effective_calls += 1
self.statistic.crashes += 1
self.debug(scratchfile, "segfault", solver_cli, stdout, stderr, random_string())
logging.info(str(i)+"/"+str(self.args.iterations)+ " Segfault! Stop testing on this seed.")
_,path = self.report(scratchfile, "segfault", solver_cli, stdout, stderr, random_string())
logging.debug(str(i)+"/"+str(self.args.iterations)+ " Segfault! Stop testing on this seed.")
logging.info(bcolors.BOLD+bcolors.WARNING+"Detected segfault: "+ path+bcolors.ENDC)
return False # stop testing

elif exitcode == 137: #timeout
Expand Down Expand Up @@ -268,16 +292,17 @@ def test(self, formula,i):
# non-erroneous solver runs (opfuzz) for soundness bugs.
if not oracle.equals(result):
self.statistic.soundness += 1
self.report(scratchfile, "incorrect", solver_cli, stdout, stderr, random_string())
_,path = self.report(scratchfile, "incorrect", solver_cli, stdout, stderr, random_string())
logging.debug(str(i)+"/"+str(self.args.iterations)+ " Soundness bug! Stop testing on this seed.")
logging.info(bcolors.BOLD+bcolors.WARNING+"Detected soundness bug! "+path+bcolors.ENDC)

if reference:
# Produce a diff bug report for soundness bugs in
# the opfuzz case
ref_cli = reference[0]
ref_stdout = reference[1]
ref_stderr = reference[2]
self.report_diff(scratchfile, "incorrect",
path = self.report_diff(scratchfile, "incorrect",
ref_cli, ref_stdout, ref_stderr,
solver_cli, stdout, stderr,
random_string())
Expand Down Expand Up @@ -308,7 +333,7 @@ def report(self, scratchfile, bugtype, cli, stdout, stderr, report_id):
log.write(stderr)
log.write("stdout:\n")
log.write(stdout)
return report_id
return report_id, report

def report_diff(self, scratchfile, bugtype,
ref_cli, ref_stdout, ref_stderr,
Expand All @@ -335,7 +360,7 @@ def report_diff(self, scratchfile, bugtype,
log.write(sol_stderr)
log.write("stdout:\n")
log.write(sol_stdout)
return report_id
return report_id, report


def __del__(self):
Expand All @@ -344,5 +369,5 @@ def __del__(self):
if self.args.name in file:
os.remove(os.path.join(self.args.scratchfolder, file))

if not self.args.quiet:
self.statistic.printsum()
# if not self.args.quiet:
# self.statistic.printsum()
4 changes: 2 additions & 2 deletions src/modules/Solver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ def solve(self, file, timeout, debug=False):
stdout = ""
stderr = ""
return stdout, stderr, 137
except KeyboardInterrupt:
exit(0)
# except KeyboardInterrupt:
# exit(0)
except ValueError as e:
print("Subprocess bug.")
stdout = ""
Expand Down
43 changes: 24 additions & 19 deletions src/modules/Statistic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,35 +4,40 @@
class Statistic:

def __init__(self):
# print("Yin-Yang is running:")
self.starttime = time.time()
self.seeds = 0
self.total_seeds = 0
self.invalid_seeds = 0
self.total_generations = 0
self.unsuccessful_generations = 0
self.mutants = 0
self.invalid_mutants = 0
self.crashes= 0
self.soundness = 0
self.duplicates = 0
self.timeout = 0
self.ignored = 0
self.solver_calls = 0
self.effective_calls = 0

def printbar(self, start_time):
total_time = time.time() - start_time
bar="Performed %d solver calls (%d calls/s, eff: %d, %d mutants/s)" %(self.solver_calls, self.solver_calls / total_time, float(self.effective_calls // self.solver_calls),self.mutants/total_time)
logging.warning(bar)
if self.solver_calls != 0:
eff = round((float(self.effective_calls) / float(self.solver_calls))*100, 1)
eff_str = str(eff) + "%"
else:
eff_str = "NaN"

def printsum(self):
summary = """
solver_calls_per_sec = round(float(self.solver_calls) / float(total_time), 1)
solver_calls_per_sec_str= round(float(self.solver_calls) / float(total_time), 1)

mutants_per_sec = round(float(self.mutants) / float(total_time), 1)
mutants_per_sec_str = str(mutants_per_sec)
bar="Performed %d solver calls (%s calls/s, eff: %s, %s mutants/s)"\
%(self.solver_calls, solver_calls_per_sec_str, eff_str, mutants_per_sec_str)
logging.info(bar)

Summary:
Passed time: %ds
Generated mutants: %d
Used seeds: %d
Crash issues: %d
Soundness issues: %d
Duplicate issues: %d
Timeout cases: %d
Ignored issues: %d
""" \
% (time.time()-self.starttime, self.mutants, self.seeds, self.crashes, self.soundness, self.duplicates, self.timeout, self.ignored)
# print(summary, end="\n", flush=True)
def printsum(self):
valid_seeds = self.total_seeds - self.invalid_seeds
num_bugs = self.crashes + self.soundness
summary = "\b\b\r%d seeds processed, %d valid, %d invalid \n%d bug triggers found"\
%(self.total_seeds, valid_seeds, self.invalid_seeds, num_bugs)
print(summary)
2 changes: 1 addition & 1 deletion tests/integration_tests/opfuzz/sanity_opfuzz.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

def call_fuzzer(first_config, second_config, fn, opts):
cmd = python+' yinyang.py '+ '"'+ first_config+ ";" + second_config + '" ' + opts + ' ' + fn
# print(cmd)
print(cmd)
output = subprocess.getoutput(cmd)
soundness_issues=None
crash_issues = None
Expand Down
10 changes: 10 additions & 0 deletions yinyang.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
#! /usr/bin/python3.7
import sys
import signal
import logging

from src.args import args
from src.modules.Fuzzer import Fuzzer

def control_c(sig, frame):
print("\b\b\rUser interrupt")
fuzzer.statistic.printsum()
sys.exit(0)

signal.signal(signal.SIGINT,control_c)
fuzzer = Fuzzer(args)
fuzzer.run()

0 comments on commit bb19017

Please sign in to comment.