diff --git a/src/modules/Fuzzer.py b/src/modules/Fuzzer.py index 6e8e023..e13cb08 100644 --- a/src/modules/Fuzzer.py +++ b/src/modules/Fuzzer.py @@ -21,6 +21,17 @@ from src.generators.TypeAwareOpMutation import TypeAwareOpMutation from src.generators.SemanticFusion.SemanticFusion import SemanticFusion +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[91m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + class Fuzzer: def __init__(self, args): @@ -31,6 +42,7 @@ def __init__(self, args): self.generator = None self.old_time = time.time() self.start_time = time.time() + self.first_status_bar_printed = False # Init logging log_fn = self.args.logfolder+"/"+str(self.args.name) @@ -72,18 +84,19 @@ def run(self): seed = seeds.pop(random.randrange(len(seeds))) logging.debug("Processing seed "+seed) + self.statistic.total_seeds += 1 - self.statistic.seeds += 1 if not self.admissible_seed_size(seed): + self.statistic.invalid_seeds += 1 + logging.debug("Skipping invalid seed: exceeds max file size") - self.statistic.ignored += 1 continue self.currentseeds = Path(seed).stem script = parse_file(seed,silent=True) if not script: # i.e. parsing was unsucessful - self.statistic.ignored += 1 + self.statistic.invalid_seeds += 1 logging.debug("Skipping invalid seed: error in parsing") continue @@ -120,21 +133,30 @@ def run(self): unsuccessful=0 for i in range(self.args.iterations): if not self.args.quiet: + if not self.first_status_bar_printed and time.time() - self.old_time >= 1: + self.statistic.printbar(self.start_time) + self.old_time = time.time() + self.first_status_bar_printed = True + + if time.time() - self.old_time >= 5.0: self.statistic.printbar(self.start_time) self.old_time = time.time() formula, success, skip_seed = self.generator.generate() if not success: + self.statistic.unsuccessful_generations += 1 unsuccessful+=1 continue - if not self.test(formula,i): break + if not self.test(formula,i+1): break self.statistic.mutants += 1 if skip_seed: break - successful= self.args.iterations - unsuccessful + successful = self.args.iterations - unsuccessful logging.debug("Finished generations: "+ str(successful)+" successful, "+ str(unsuccessful)+ " unsuccessful") + print("All seeds processed") + self.statistic.printsum() def create_testbook(self, formula): testbook = [] @@ -192,7 +214,7 @@ def init_oracle(self): assert(False) - def test(self, formula,i): + def test(self, formula, i): """ Tests the solvers with the formula returning "False" if the testing on formula should be stopped and "True" otherwise. @@ -212,10 +234,11 @@ def test(self, formula,i): # (2) Match against the duplicate list to avoid reporting duplicate bugs. if not self.in_duplicate_list(stdout, stderr): - self.statistic.effective_call += 1 + self.statistic.effective_calls += 1 self.statistic.crashes += 1 - self.report(scratchfile, "crash", solver_cli, stdout, stderr, random_string()) + _,path = self.report(scratchfile, "crash", solver_cli, stdout, stderr, random_string()) logging.debug("Crash! Stop testing on this seed.") + logging.info(bcolors.BOLD+bcolors.WARNING+"Detected crash bug: "+ path+bcolors.ENDC) else: self.statistic.duplicates += 1 logging.debug("Duplicate. Stop testing on this seed.") @@ -225,16 +248,17 @@ def test(self, formula,i): # the ignore list. if self.in_ignore_list(stdout, stderr): logging.debug("Invalid mutant:ignore_list. solver="+str(solver_cli)) - self.statistic.ignored += 1 + self.statistic.invalid_mutants+= 1 continue # continue with next solver (4) # (3b) Check whether the exit code is nonzero. if exitcode != 0: if exitcode == -signal.SIGSEGV or exitcode == 245: #segfault - self.statistic.effective_call += 1 + self.statistic.effective_calls += 1 self.statistic.crashes += 1 - self.debug(scratchfile, "segfault", solver_cli, stdout, stderr, random_string()) - logging.info(str(i)+"/"+str(self.args.iterations)+ " Segfault! Stop testing on this seed.") + _,path = self.report(scratchfile, "segfault", solver_cli, stdout, stderr, random_string()) + logging.debug(str(i)+"/"+str(self.args.iterations)+ " Segfault! Stop testing on this seed.") + logging.info(bcolors.BOLD+bcolors.WARNING+"Detected segfault: "+ path+bcolors.ENDC) return False # stop testing elif exitcode == 137: #timeout @@ -268,8 +292,9 @@ def test(self, formula,i): # non-erroneous solver runs (opfuzz) for soundness bugs. if not oracle.equals(result): self.statistic.soundness += 1 - self.report(scratchfile, "incorrect", solver_cli, stdout, stderr, random_string()) + _,path = self.report(scratchfile, "incorrect", solver_cli, stdout, stderr, random_string()) logging.debug(str(i)+"/"+str(self.args.iterations)+ " Soundness bug! Stop testing on this seed.") + logging.info(bcolors.BOLD+bcolors.WARNING+"Detected soundness bug! "+path+bcolors.ENDC) if reference: # Produce a diff bug report for soundness bugs in @@ -277,7 +302,7 @@ def test(self, formula,i): ref_cli = reference[0] ref_stdout = reference[1] ref_stderr = reference[2] - self.report_diff(scratchfile, "incorrect", + path = self.report_diff(scratchfile, "incorrect", ref_cli, ref_stdout, ref_stderr, solver_cli, stdout, stderr, random_string()) @@ -308,7 +333,7 @@ def report(self, scratchfile, bugtype, cli, stdout, stderr, report_id): log.write(stderr) log.write("stdout:\n") log.write(stdout) - return report_id + return report_id, report def report_diff(self, scratchfile, bugtype, ref_cli, ref_stdout, ref_stderr, @@ -335,7 +360,7 @@ def report_diff(self, scratchfile, bugtype, log.write(sol_stderr) log.write("stdout:\n") log.write(sol_stdout) - return report_id + return report_id, report def __del__(self): @@ -344,5 +369,5 @@ def __del__(self): if self.args.name in file: os.remove(os.path.join(self.args.scratchfolder, file)) - if not self.args.quiet: - self.statistic.printsum() + # if not self.args.quiet: + # self.statistic.printsum() diff --git a/src/modules/Solver.py b/src/modules/Solver.py index 21ba5ae..adc3ac7 100644 --- a/src/modules/Solver.py +++ b/src/modules/Solver.py @@ -70,8 +70,8 @@ def solve(self, file, timeout, debug=False): stdout = "" stderr = "" return stdout, stderr, 137 - except KeyboardInterrupt: - exit(0) + # except KeyboardInterrupt: + # exit(0) except ValueError as e: print("Subprocess bug.") stdout = "" diff --git a/src/modules/Statistic.py b/src/modules/Statistic.py index 536a572..327afa5 100644 --- a/src/modules/Statistic.py +++ b/src/modules/Statistic.py @@ -4,35 +4,40 @@ class Statistic: def __init__(self): - # print("Yin-Yang is running:") self.starttime = time.time() - self.seeds = 0 + self.total_seeds = 0 + self.invalid_seeds = 0 + self.total_generations = 0 + self.unsuccessful_generations = 0 self.mutants = 0 + self.invalid_mutants = 0 self.crashes= 0 self.soundness = 0 self.duplicates = 0 self.timeout = 0 - self.ignored = 0 self.solver_calls = 0 self.effective_calls = 0 def printbar(self, start_time): total_time = time.time() - start_time - bar="Performed %d solver calls (%d calls/s, eff: %d, %d mutants/s)" %(self.solver_calls, self.solver_calls / total_time, float(self.effective_calls // self.solver_calls),self.mutants/total_time) - logging.warning(bar) + if self.solver_calls != 0: + eff = round((float(self.effective_calls) / float(self.solver_calls))*100, 1) + eff_str = str(eff) + "%" + else: + eff_str = "NaN" - def printsum(self): - summary = """ + solver_calls_per_sec = round(float(self.solver_calls) / float(total_time), 1) + solver_calls_per_sec_str= round(float(self.solver_calls) / float(total_time), 1) + + mutants_per_sec = round(float(self.mutants) / float(total_time), 1) + mutants_per_sec_str = str(mutants_per_sec) + bar="Performed %d solver calls (%s calls/s, eff: %s, %s mutants/s)"\ + %(self.solver_calls, solver_calls_per_sec_str, eff_str, mutants_per_sec_str) + logging.info(bar) -Summary: -Passed time: %ds -Generated mutants: %d -Used seeds: %d -Crash issues: %d -Soundness issues: %d -Duplicate issues: %d -Timeout cases: %d -Ignored issues: %d -""" \ - % (time.time()-self.starttime, self.mutants, self.seeds, self.crashes, self.soundness, self.duplicates, self.timeout, self.ignored) - # print(summary, end="\n", flush=True) + def printsum(self): + valid_seeds = self.total_seeds - self.invalid_seeds + num_bugs = self.crashes + self.soundness + summary = "\b\b\r%d seeds processed, %d valid, %d invalid \n%d bug triggers found"\ + %(self.total_seeds, valid_seeds, self.invalid_seeds, num_bugs) + print(summary) diff --git a/tests/integration_tests/opfuzz/sanity_opfuzz.py b/tests/integration_tests/opfuzz/sanity_opfuzz.py index 8415a75..109c527 100644 --- a/tests/integration_tests/opfuzz/sanity_opfuzz.py +++ b/tests/integration_tests/opfuzz/sanity_opfuzz.py @@ -5,7 +5,7 @@ def call_fuzzer(first_config, second_config, fn, opts): cmd = python+' yinyang.py '+ '"'+ first_config+ ";" + second_config + '" ' + opts + ' ' + fn - # print(cmd) + print(cmd) output = subprocess.getoutput(cmd) soundness_issues=None crash_issues = None diff --git a/yinyang.py b/yinyang.py index 8606f0d..87db6c0 100755 --- a/yinyang.py +++ b/yinyang.py @@ -1,7 +1,17 @@ #! /usr/bin/python3.7 import sys +import signal +import logging from src.args import args from src.modules.Fuzzer import Fuzzer + +def control_c(sig, frame): + print("\b\b\rUser interrupt") + fuzzer.statistic.printsum() + sys.exit(0) + +signal.signal(signal.SIGINT,control_c) fuzzer = Fuzzer(args) fuzzer.run() +