From e595f20b2de3e87736fc83ac90b96e91b768ddfc Mon Sep 17 00:00:00 2001 From: Theodoros Theodoridis Date: Wed, 11 Jan 2023 16:34:58 +0100 Subject: [PATCH] [dead2] Add basic reductions --- dead/__init__.py | 24 ++++- dead/differential_testing.py | 53 +++++++---- dead/output.py | 55 +++++++++++- dead/reduction.py | 168 +++++++++++++++++++++++++++++++++++ setup.cfg | 6 +- 5 files changed, 281 insertions(+), 25 deletions(-) create mode 100644 dead/reduction.py diff --git a/dead/__init__.py b/dead/__init__.py index c03f709..0599334 100644 --- a/dead/__init__.py +++ b/dead/__init__.py @@ -7,6 +7,7 @@ from dead.config import dump_config, interactive_init from dead.differential_testing import DifferentialTestingMode, generate_and_test from dead.output import write_cases_to_directory +from dead.reduction import reduce_case def __arg_to_compiler_exe(arg: str | None) -> CompilerExe | None: @@ -84,7 +85,6 @@ def parse_args() -> argparse.Namespace: action=argparse.BooleanOptionalAction, help="Make the temporary configuration overrides permanent.", ) - parser.add_argument( "compilation_command1", type=str, @@ -107,6 +107,12 @@ def parse_args() -> argparse.Namespace: "first command eliminated, in the bidirectional mode (default) a case is " "interesting as long as at least one command misses a marker", ) + + parser.add_argument( + "--reduce", + action=argparse.BooleanOptionalAction, + help="Also reduce the discovered cases", + ) parser.add_argument( "--jobs", "-j", @@ -148,7 +154,19 @@ def run_as_module() -> None: setting1, setting2, __arg_to_testing_mode(args.testing_mode), - args.number_attempts, + args.number_candidates, args.jobs, ) - write_cases_to_directory(cases, args.output_directory) + reductions = {} + if args.reduce: + for case in cases: + # XXX: how to a select marker? + target_marker = ( + case.markers_only_eliminated_by_setting1 + + case.markers_only_eliminated_by_setting2 + )[0] + reduction = reduce_case(case, target_marker, args.jobs) + assert reduction + reductions[case] = reduction + + write_cases_to_directory(cases, reductions, args.output_directory) diff --git a/dead/differential_testing.py b/dead/differential_testing.py index 4419063..44bfc76 100644 --- a/dead/differential_testing.py +++ b/dead/differential_testing.py @@ -51,17 +51,25 @@ class DifferentialTestingCase: markers_only_eliminated_by_setting1: tuple[DCEMarker | VRMarker, ...] markers_only_eliminated_by_setting2: tuple[DCEMarker | VRMarker, ...] + def __post_init__(self) -> None: + assert set(self.markers_only_eliminated_by_setting1).isdisjoint( + self.markers_only_eliminated_by_setting2 + ) + class DifferentialTestingMode(Enum): """ - - Unidirectional: a marker is interesting only if the first - compilation setting missed it and the second eliminated - - Bidirectional: a marker is interesting if any of the compilation settings - missed it and the other found it + - Unidirectional: any marker is interesting only if the first + compilation setting missed it and the second eliminated it + - Bidirectional: any marker is interesting if any of the compilation + settings missed it and the other eliminated it + - MarkerMissedByFirst: a particular marker is interesting if the + first compilation setting missed it and the other eliminated it """ - Unidirectional = 0 + Unidirectional = 0 # AnyMissedByFirst? Bidirectional = 1 + MarkerMissedByFirst = 2 def differential_test( @@ -69,6 +77,7 @@ def differential_test( setting1: CompilationSetting, setting2: CompilationSetting, testing_mode: DifferentialTestingMode = DifferentialTestingMode.Bidirectional, + missed_marker: DCEMarker | VRMarker | None = None, ) -> DifferentialTestingCase | None: """Instrument `program`, compile it with `setting1` and `setting2` and check if the set of eliminated markers differ. @@ -87,11 +96,16 @@ def differential_test( setting2 (CompilationSetting): the second compilation setting with which to compile the instrumented program - testing_direction (DifferentialTestingDirection): + testing_mode (DifferentialTestingMode): whether to accept cases whether where any of the two settings miss at least one marker (Bidirectional), or cases where markers are - eliminated by `setting1` and eliminated by `setting2` - + missed by `setting1` and eliminated by `setting2` (Unidirectional). + In MarkerMissedByFirst mode, if `missed_marker` is not + missed by the First setting and eliminated by the other, + the case is not interesting and None is returned. + missed_marker (DCEMarker | VRMarker | None): + If `testing_mode` is MarkerMissecByFirst, only `missed_marker` is + checked: it must be missed by the first setting and found by the other. Returns: (DifferentialTestingCase | None): interesting case if found @@ -103,7 +117,9 @@ def differential_test( # Instrument program try: - instr_program = instrument_program(program) + instr_program = instrument_program( + setting1.preprocess_program(program, make_compiler_agnostic=True) + ) except AssertionError: return None @@ -114,13 +130,17 @@ def differential_test( only_eliminated_by_setting2 = tuple(dead_markers2 - dead_markers1) # Is the candidate interesting? - if not only_eliminated_by_setting1 and not only_eliminated_by_setting2: - return None - if testing_mode == DifferentialTestingMode.Unidirectional: - if not only_eliminated_by_setting1: - return None - else: - assert testing_mode == DifferentialTestingMode.Bidirectional + match testing_mode: + case DifferentialTestingMode.Bidirectional: + if not only_eliminated_by_setting1 and not only_eliminated_by_setting2: + return None + case DifferentialTestingMode.Unidirectional: + if not only_eliminated_by_setting1: + return None + case DifferentialTestingMode.MarkerMissedByFirst: + assert missed_marker + if missed_marker not in only_eliminated_by_setting2: + return None return DifferentialTestingCase( program=instr_program, @@ -131,6 +151,7 @@ def differential_test( ) +# XXX: does this really belong in this module? def generate_and_test( setting1: CompilationSetting, setting2: CompilationSetting, diff --git a/dead/output.py b/dead/output.py index 20920b8..1c21490 100644 --- a/dead/output.py +++ b/dead/output.py @@ -4,10 +4,51 @@ from diopter.compiler import CompilationOutput, CompilationOutputKind from dead.differential_testing import DifferentialTestingCase +from dead.reduction import Reduction + + +def write_reduction_to_directory( + reduction: Reduction | None, output_directory: Path +) -> None: + if not reduction: + return + reduction_dir = output_directory / "reduction" + reduction_dir.mkdir() + code_file = (reduction_dir / "reduced_code").with_suffix( + reduction.reduced_program.language.to_suffix() + ) + + with open(code_file, "w") as f: + print(reduction.reduced_program.code, file=f) + + with open(reduction_dir / "good_setting", "w") as f: + print( + " ".join( + reduction.good_setting.get_compilation_cmd( + (reduction.reduced_program, Path(code_file.name)), + CompilationOutput(Path("dummy1.s"), CompilationOutputKind.Assembly), + ) + ), + file=f, + ) + + with open(reduction_dir / "bad_setting", "w") as f: + print( + " ".join( + reduction.bad_setting.get_compilation_cmd( + (reduction.reduced_program, Path(code_file.name)), + CompilationOutput(Path("dummy1.s"), CompilationOutputKind.Assembly), + ) + ), + file=f, + ) + + with open(reduction_dir / "target_marker", "w") as f: + print(reduction.target_marker.to_macro(), file=f) def write_case_to_directory( - case: DifferentialTestingCase, output_directory: Path + case: DifferentialTestingCase, reduction: Reduction | None, output_directory: Path ) -> None: output_directory.mkdir(parents=True, exist_ok=True) code_file = (output_directory / "code").with_suffix( @@ -37,6 +78,7 @@ def write_case_to_directory( ), file=f, ) + with open(output_directory / "markers_only_eliminated_by_setting1", "w") as f: print( "\n".join( @@ -52,14 +94,21 @@ def write_case_to_directory( ), file=f, ) + write_reduction_to_directory(reduction, output_directory) def write_cases_to_directory( - cases: Sequence[DifferentialTestingCase], output_directory: Path + cases: Sequence[DifferentialTestingCase], + reductions: dict[DifferentialTestingCase, Reduction], + output_directory: Path, ) -> None: output_directory.mkdir(parents=True, exist_ok=True) output_sub_dir_n = 0 for case in cases: while (output_directory / str(output_sub_dir_n)).exists(): output_sub_dir_n += 1 - write_case_to_directory(case, output_directory / str(output_sub_dir_n)) + write_case_to_directory( + case, + reductions[case] if case in reductions else None, + output_directory / str(output_sub_dir_n), + ) diff --git a/dead/reduction.py b/dead/reduction.py new file mode 100644 index 0000000..576e01a --- /dev/null +++ b/dead/reduction.py @@ -0,0 +1,168 @@ +""" +Implements the reduction logic used for generating minimal test cases + +e.g., given a DifferentialTestingCase and a target marker, which should +still be present in the minimal case, it produces a Reduction + +reduced_case = reduce_case( case, target_marker, jobs=128) + +reduce_case.reduced_program # the reduced program + +""" + +from dataclasses import dataclass + +from dead_instrumenter.instrumenter import ( + DCEMarker, + InstrumentedProgram, + VRMarker, + annotate_with_static, +) +from diopter.compiler import CompilationSetting, SourceProgram +from diopter.reducer import Reducer, ReductionCallback +from diopter.sanitizer import Sanitizer + +from callchain_checker.callchain_checker import callchain_exists +from dead.config import DeadConfig +from dead.differential_testing import DifferentialTestingCase + + +@dataclass(frozen=True, kw_only=True) +class DeadReductionCallback(ReductionCallback): + """Callback implementing the interestingness test for creduce. + + Attributes: + bad_setting (CompilationSetting): + the setting that should misse the `target_marker` + good_setting (CompilationSetting): + the setting that should eliminate the `target_marker` + target_marker (VRMarker | DCEMarker): + the marker targeted by the reduction + sanitizer (Sanitizer): + sanitizer used for checking that the reduced program is valid + """ + + bad_setting: CompilationSetting + good_setting: CompilationSetting + target_marker: DCEMarker | VRMarker + sanitizer: Sanitizer + + def test(self, program: SourceProgram) -> bool: + """Reduction test + Args: + program (SourceProgram): + the reduced program that is being checked + Returns: + bool: + whether the reduced program should be kept + """ + assert isinstance(program, InstrumentedProgram) + + # Check that the marker can potentially be called from main + if not callchain_exists(program, "main", self.target_marker.to_macro()): + return False + + # creduce may drop static annotations, add them back + program = annotate_with_static(program) + assert isinstance(program, InstrumentedProgram) + + # The bad setting should miss the marker + if self.target_marker in program.find_dead_markers(self.bad_setting): + return False + + # The good setting should eliminate the marker + if self.target_marker not in program.find_dead_markers(self.good_setting): + return False + + # Sanitize + return bool( + self.sanitizer.sanitize(program.disable_remaining_markers(), debug=False) + ) + + +@dataclass(frozen=True, kw_only=True) +class Reduction: + """A reduction represents a differential testing case and a reduced + program containing the target marker that is missed by the bad setting + but is eliminated by the good setting. + + Attributes: + case (DifferentialTestingCase): + the case in which the `target_marker` is eliminated by + `good_setting` but missed by `bad_setting` + reduced_program (InstrumentedProgram): + a reduced version of `case.program` that still contains the + `target_marker` which is eliminated by `good_setting` and + missed by `bad_setting` + target_marker (VRMarker | DCEMarker): + the marker targeted by the reduction + bad_setting (CompilationSetting): + the setting that misses the `target_marker` both + in `case.program` and in `reduced_program` + good_setting (CompilationSetting): + the setting that eliminates the `target_marker` both + in `case.program` and in `reduced_program` + """ + + case: DifferentialTestingCase + reduced_program: InstrumentedProgram + target_marker: VRMarker | DCEMarker + bad_setting: CompilationSetting + good_setting: CompilationSetting + + +def reduce_case( + case: DifferentialTestingCase, target_marker: DCEMarker | VRMarker, jobs: int +) -> Reduction: + """Reduces `case.program` such that `target_marker` is missed by + case.setting1 or case.setting2 and missed by the other. + + Args: + case (DifferentialTestingCase): + the case to be reduced + target_marker (DCEMarker | VRMarker): + the marker that will still be present in the reduced program + jobs (int): + how many parallel jobs to use + + Returns: + Reduction: + reduced case containing `target_marker` + """ + + # Figure out which setting misses and which eliminates the marker + if target_marker in case.markers_only_eliminated_by_setting1: + good_setting = case.setting1 + bad_setting = case.setting2 + else: + assert target_marker in case.markers_only_eliminated_by_setting2 + good_setting = case.setting2 + bad_setting = case.setting1 + + # Setup reducer and sanitizer + config = DeadConfig.get_config() + san = Sanitizer(gcc=config.gcc, clang=config.clang, ccomp=config.ccomp) + reducer = Reducer(str(config.creduce)) + + # Reduce + reduced_program = reducer.reduce( + case.program, + DeadReductionCallback( + bad_setting=bad_setting, + good_setting=good_setting, + sanitizer=san, + target_marker=target_marker, + ), + jobs=jobs, + debug=False, + ) + assert isinstance(reduced_program, InstrumentedProgram) + reduced_program = annotate_with_static(reduced_program) + assert isinstance(reduced_program, InstrumentedProgram) + return Reduction( + case=case, + reduced_program=reduced_program, + target_marker=target_marker, + bad_setting=bad_setting, + good_setting=good_setting, + ) diff --git a/setup.cfg b/setup.cfg index 26ad444..d4adee2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,9 +16,9 @@ classifiers = packages=find: python_requires = >= 3.10 install_requires = - diopter == 0.0.8 - dead_instrumenter == 0.3.0 - callchain-checker == 0.0.1 + diopter == 0.0.10 + dead_instrumenter == 0.3.1 + callchain-checker == 0.0.2 [options.packages.find] where=dead