From e3365a7f95b129f866d3f5b7ed760b73c56479dc Mon Sep 17 00:00:00 2001 From: bohendo Date: Tue, 30 Jan 2024 13:06:18 -0500 Subject: [PATCH 01/42] remove newline separators in mutation output --- slither/tools/mutator/utils/testing_generated_mutant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 4c51b7e5a..fa689f252 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -85,7 +85,7 @@ def test_patch( # pylint: disable=too-many-arguments create_mutant_file(file, index, generator_name) print( green( - f"String '{patch['old_string']}' replaced with '{patch['new_string']}' at line no. '{patch['line_number']}' ---> VALID\n" + f"String '{patch['old_string']}' replaced with '{patch['new_string']}' at line no. '{patch['line_number']}' ---> VALID" ) ) return True @@ -94,7 +94,7 @@ def test_patch( # pylint: disable=too-many-arguments if verbose: print( red( - f"String '{patch['old_string']}' replaced with '{patch['new_string']}' at line no. '{patch['line_number']}' ---> INVALID\n" + f"String '{patch['old_string']}' replaced with '{patch['new_string']}' at line no. '{patch['line_number']}' ---> INVALID" ) ) return False From 007789ff928d0bb6a4f20e9a8f028b8bc7f0cd48 Mon Sep 17 00:00:00 2001 From: bohendo Date: Tue, 30 Jan 2024 14:18:18 -0500 Subject: [PATCH 02/42] count valid RR ad CR mutants --- slither/tools/mutator/__main__.py | 15 ++++++++++---- .../mutator/mutators/abstract_mutator.py | 20 +++++++++++++++---- .../mutator/utils/testing_generated_mutant.py | 8 ++++---- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 5c13d7aea..e3fb4b990 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -196,7 +196,11 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t # total count of mutants total_count = 0 # count of valid mutants - v_count = 0 + valid_count = 0 + # count of valid revert mutants + RR_count = 0 + # count of valid comment mutants + CR_count = 0 # lines those need not be mutated (taken from RR and CR) dont_mutate_lines = [] @@ -212,6 +216,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t if contract_instance == "": logger.error("Can't find the contract") else: + logger.info(yellow(f"\nMutating contract {contract_instance}")) for M in mutators_list: m = M( compilation_unit_of_main_file, @@ -224,8 +229,10 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t output_folder, dont_mutate_lines, ) - (count_valid, count_invalid, lines_list) = m.mutate() - v_count += count_valid + (count_invalid, count_valid, count_valid_rr, count_valid_cr, lines_list) = m.mutate() + valid_count += count_valid + RR_count += count_valid_rr + CR_count += count_valid_cr total_count += count_valid + count_invalid dont_mutate_lines = lines_list if not quick_flag: @@ -244,7 +251,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t # output logger.info( yellow( - f"Done mutating, '{filename}'. Valid mutant count: '{v_count}' and Total mutant count '{total_count}'.\n" + f"Done mutating, '{filename}'. Total mutants: {total_count}, Total Valid: '{valid_count}', Valid reverts: {RR_count}, Valid comments: {CR_count}.\n" ) ) diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index 375af1e6f..d264c47c4 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -19,8 +19,10 @@ class AbstractMutator( ): # pylint: disable=too-few-public-methods,too-many-instance-attributes NAME = "" HELP = "" - VALID_MUTANTS_COUNT = 0 INVALID_MUTANTS_COUNT = 0 + VALID_MUTANTS_COUNT = 0 + VALID_RR_MUTANTS_COUNT = 0 + VALID_CR_MUTANTS_COUNT = 0 def __init__( # pylint: disable=too-many-arguments self, @@ -71,7 +73,7 @@ def _mutate(self) -> Dict: """TODO Documentation""" return {} - def mutate(self) -> Tuple[int, int, List[int]]: + def mutate(self) -> Tuple[int, int, int, int, List[int]]: # call _mutate function from different mutators (all_patches) = self._mutate() if "patches" not in all_patches: @@ -97,11 +99,18 @@ def mutate(self) -> Tuple[int, int, List[int]]: ) # if RR or CR and valid mutant, add line no. if self.NAME in ("RR", "CR") and flag: + if self.NAME == 'RR': + self.VALID_RR_MUTANTS_COUNT += 1 + if self.NAME == 'CR': + self.VALID_CR_MUTANTS_COUNT += 1 + logger.info(yellow("Severe mutant is valid, skipping further mutations")) self.dont_mutate_line.append(patch["line_number"]) + # count the valid and invalid mutants if not flag: self.INVALID_MUTANTS_COUNT += 1 continue + logger.info(yellow("Severe mutant is invalid, continuing further mutations")) self.VALID_MUTANTS_COUNT += 1 patched_txt, _ = apply_patch(original_txt, patch, 0) diff = create_diff(self.compilation_unit, original_txt, patched_txt, file) @@ -113,8 +122,11 @@ def mutate(self) -> Tuple[int, int, List[int]]: self.output_folder + "/patches_file.txt", "a", encoding="utf8" ) as patches_file: patches_file.write(diff + "\n") + # logger.info(yellow(f"{self.VALID_MUTANTS_COUNT} valid mutants, {self.INVALID_MUTANTS_COUNT} invalid mutants")) return ( - self.VALID_MUTANTS_COUNT, self.INVALID_MUTANTS_COUNT, - self.dont_mutate_line, + self.VALID_MUTANTS_COUNT, + self.VALID_RR_MUTANTS_COUNT, + self.VALID_CR_MUTANTS_COUNT, + self.dont_mutate_line ) diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index fa689f252..969d87dcc 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -83,18 +83,18 @@ def test_patch( # pylint: disable=too-many-arguments if compile_generated_mutant(file, mappings): if run_test_cmd(command, file, timeout): create_mutant_file(file, index, generator_name) - print( + logger.info( green( - f"String '{patch['old_string']}' replaced with '{patch['new_string']}' at line no. '{patch['line_number']}' ---> VALID" + f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> VALID" ) ) return True reset_file(file) if verbose: - print( + logger.info( red( - f"String '{patch['old_string']}' replaced with '{patch['new_string']}' at line no. '{patch['line_number']}' ---> INVALID" + f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> INVALID" ) ) return False From 3c468b6eaf72ea46ae8f8147abb468245cc0e66c Mon Sep 17 00:00:00 2001 From: bohendo Date: Tue, 30 Jan 2024 16:27:13 -0500 Subject: [PATCH 03/42] count different categories of total mutants --- .../mutator/mutators/abstract_mutator.py | 72 +++++++++++-------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index d264c47c4..9df7c45ab 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -23,6 +23,10 @@ class AbstractMutator( VALID_MUTANTS_COUNT = 0 VALID_RR_MUTANTS_COUNT = 0 VALID_CR_MUTANTS_COUNT = 0 + # total revert/comment/tweak mutants that were generated and compiled + total_mutant_counts = [0, 0, 0] + # total valid revert/comment/tweak mutants + valid_mutant_counts = [0, 0, 0] def __init__( # pylint: disable=too-many-arguments self, @@ -73,12 +77,12 @@ def _mutate(self) -> Dict: """TODO Documentation""" return {} - def mutate(self) -> Tuple[int, int, int, int, List[int]]: + def mutate(self) -> Tuple[List[int], List[int], List[int]]: # call _mutate function from different mutators (all_patches) = self._mutate() if "patches" not in all_patches: logger.debug("No patches found by %s", self.NAME) - return (0, 0, self.dont_mutate_line) + return ([0,0,0], [0,0,0], self.dont_mutate_line) for file in all_patches["patches"]: original_txt = self.slither.source_code[file].encode("utf8") @@ -87,7 +91,8 @@ def mutate(self) -> Tuple[int, int, int, int, List[int]]: logger.info(yellow(f"Mutating {file} with {self.NAME} \n")) for patch in patches: # test the patch - flag = test_patch( + + patchIsValid = test_patch( file, patch, self.test_command, @@ -97,36 +102,41 @@ def mutate(self) -> Tuple[int, int, int, int, List[int]]: self.solc_remappings, self.verbose, ) - # if RR or CR and valid mutant, add line no. - if self.NAME in ("RR", "CR") and flag: + + # count the valid mutants, flag RR/CR mutants to skip further mutations + if patchIsValid == 0: if self.NAME == 'RR': - self.VALID_RR_MUTANTS_COUNT += 1 - if self.NAME == 'CR': - self.VALID_CR_MUTANTS_COUNT += 1 - logger.info(yellow("Severe mutant is valid, skipping further mutations")) - self.dont_mutate_line.append(patch["line_number"]) - - # count the valid and invalid mutants - if not flag: - self.INVALID_MUTANTS_COUNT += 1 - continue - logger.info(yellow("Severe mutant is invalid, continuing further mutations")) - self.VALID_MUTANTS_COUNT += 1 - patched_txt, _ = apply_patch(original_txt, patch, 0) - diff = create_diff(self.compilation_unit, original_txt, patched_txt, file) - if not diff: - logger.info(f"Impossible to generate patch; empty {patches}") - - # add valid mutant patches to a output file - with open( - self.output_folder + "/patches_file.txt", "a", encoding="utf8" - ) as patches_file: - patches_file.write(diff + "\n") + self.valid_mutant_counts[0] += 1 + self.dont_mutate_line.append(patch['line_number']) + elif self.NAME == 'CR': + self.valid_mutant_counts[1] += 1 + self.dont_mutate_line.append(patch['line_number']) + else: + self.valid_mutant_counts[2] += 1 + + patched_txt,_ = apply_patch(original_txt, patch, 0) + diff = create_diff(self.compilation_unit, original_txt, patched_txt, file) + if not diff: + logger.info(f"Impossible to generate patch; empty {patches}") + + # add valid mutant patches to a output file + with open( + self.output_folder + "/patches_file.txt", "a", encoding="utf8" + ) as patches_file: + patches_file.write(diff + "\n") + + # count the total number of mutants that we were able to compile + if patchIsValid != 2: + if self.NAME == 'RR': + self.total_mutant_counts[0] += 1 + elif self.NAME == 'CR': + self.total_mutant_counts[1] += 1 + else: + self.total_mutant_counts[2] += 1 # logger.info(yellow(f"{self.VALID_MUTANTS_COUNT} valid mutants, {self.INVALID_MUTANTS_COUNT} invalid mutants")) + return ( - self.INVALID_MUTANTS_COUNT, - self.VALID_MUTANTS_COUNT, - self.VALID_RR_MUTANTS_COUNT, - self.VALID_CR_MUTANTS_COUNT, + self.total_mutant_counts, + self.valid_mutant_counts, self.dont_mutate_line ) From f951ec7bb38132e7f658f6cc4b25a9a24732b041 Mon Sep 17 00:00:00 2001 From: bohendo Date: Tue, 30 Jan 2024 16:46:34 -0500 Subject: [PATCH 04/42] print more detailed campaign summary --- slither/tools/mutator/__main__.py | 103 +++++++++++++++++------------- 1 file changed, 58 insertions(+), 45 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index e3fb4b990..3324950a3 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -193,50 +193,56 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t sl = Slither(filename, **vars(args)) # create a backup files files_dict = backup_source_file(sl.source_code, output_folder) - # total count of mutants - total_count = 0 - # count of valid mutants - valid_count = 0 - # count of valid revert mutants - RR_count = 0 - # count of valid comment mutants - CR_count = 0 + # total revert/comment/tweak mutants that were generated and compiled + total_mutant_counts = [0, 0, 0] + # total valid revert/comment/tweak mutants + valid_mutant_counts = [0, 0, 0] # lines those need not be mutated (taken from RR and CR) dont_mutate_lines = [] # mutation + contract_instance = '' try: for compilation_unit_of_main_file in sl.compilation_units: - contract_instance = "" for contract in compilation_unit_of_main_file.contracts: if contract_names is not None and contract.name in contract_names: contract_instance = contract + elif contract_names is not None and contract.name not in contract_names: + contract_instance = "SLITHER_SKIP_MUTATIONS" elif str(contract.name).lower() == contract_name.lower(): contract_instance = contract - if contract_instance == "": - logger.error("Can't find the contract") - else: - logger.info(yellow(f"\nMutating contract {contract_instance}")) - for M in mutators_list: - m = M( - compilation_unit_of_main_file, - int(timeout), - test_command, - test_directory, - contract_instance, - solc_remappings, - verbose, - output_folder, - dont_mutate_lines, - ) - (count_invalid, count_valid, count_valid_rr, count_valid_cr, lines_list) = m.mutate() - valid_count += count_valid - RR_count += count_valid_rr - CR_count += count_valid_cr - total_count += count_valid + count_invalid - dont_mutate_lines = lines_list - if not quick_flag: - dont_mutate_lines = [] + + if contract_instance == '': + logger.info(f"Cannot find contracts in file {filename}, try specifying them with --contract-names") + continue + + if contract_instance == 'SLITHER_SKIP_MUTATIONS': + logger.debug(f"Skipping mutations in {filename}") + continue + + logger.info(yellow(f"Mutating contract {contract_instance}")) + for M in mutators_list: + m = M( + compilation_unit_of_main_file, + int(timeout), + test_command, + test_directory, + contract_instance, + solc_remappings, + verbose, + output_folder, + dont_mutate_lines, + ) + (total_counts, valid_counts, lines_list) = m.mutate() + total_mutant_counts[0] += total_counts[0] + total_mutant_counts[1] += total_counts[1] + total_mutant_counts[2] += total_counts[2] + valid_mutant_counts[0] += valid_counts[0] + valid_mutant_counts[1] += valid_counts[1] + valid_mutant_counts[2] += valid_counts[2] + dont_mutate_lines = lines_list + if not quick_flag: + dont_mutate_lines = [] except Exception as e: # pylint: disable=broad-except logger.error(e) @@ -245,17 +251,24 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t logger.error("\nExecution interrupted by user (Ctrl + C). Cleaning up...") transfer_and_delete(files_dict) - # transfer and delete the backup files - transfer_and_delete(files_dict) - - # output - logger.info( - yellow( - f"Done mutating, '{filename}'. Total mutants: {total_count}, Total Valid: '{valid_count}', Valid reverts: {RR_count}, Valid comments: {CR_count}.\n" - ) - ) - - logger.info(magenta(f"Finished Mutation Campaign in '{args.codebase}' \n")) - + if not contract_instance == 'SLITHER_SKIP_MUTATIONS': + # transfer and delete the backup files + transfer_and_delete(files_dict) + # output + print(yellow(f"Done mutating {filename}.")) + if total_mutant_counts[0] > 0: + print(yellow(f"Revert mutants: {valid_mutant_counts[0]} valid of {total_mutant_counts[0]} ({100 * valid_mutant_counts[0]/total_mutant_counts[0]}%)")) + else: + print(yellow("Zero Revert mutants analyzed")) + + if total_mutant_counts[1] > 0: + print(yellow(f"Comment mutants: {valid_mutant_counts[1]} valid of {total_mutant_counts[1]} ({100 * valid_mutant_counts[1]/total_mutant_counts[1]}%)")) + else: + print(yellow("Zero Comment mutants analyzed")) + + if total_mutant_counts[2] > 0: + print(yellow(f"Tweak mutants: {valid_mutant_counts[2]} valid of {total_mutant_counts[2]} ({100 * valid_mutant_counts[2]/total_mutant_counts[2]}%)")) + else: + print(yellow("Zero Tweak mutants analyzed")) # endregion From a832b98c672439a55678feef9d8323861c32c9f7 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 5 Feb 2024 10:17:43 -0500 Subject: [PATCH 05/42] flag mutants that fail to compile --- .../mutator/utils/testing_generated_mutant.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 969d87dcc..9780f1d9a 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -6,7 +6,7 @@ from typing import Dict import crytic_compile from slither.tools.mutator.utils.file_handling import create_mutant_file, reset_file -from slither.utils.colors import green, red +from slither.utils.colors import green, red, yellow logger = logging.getLogger("Slither-Mutate") @@ -84,17 +84,27 @@ def test_patch( # pylint: disable=too-many-arguments if run_test_cmd(command, file, timeout): create_mutant_file(file, index, generator_name) logger.info( - green( + red( f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> VALID" ) ) - return True + reset_file(file) + return 0 # valid + else: + if verbose: + logger.info( + yellow( + f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> COMPILATION FAILURE" + ) + ) + reset_file(file) + return 2 # compile failure - reset_file(file) if verbose: logger.info( - red( + green( f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> INVALID" ) ) - return False + reset_file(file) + return 1 # invalid From bbf689642c81d167bf09dd67c7a2089ea8084755 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 5 Feb 2024 15:37:58 -0500 Subject: [PATCH 06/42] be more careful while replacing revert strings --- slither/tools/mutator/mutators/RR.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slither/tools/mutator/mutators/RR.py b/slither/tools/mutator/mutators/RR.py index e285d7a3f..ba76d657f 100644 --- a/slither/tools/mutator/mutators/RR.py +++ b/slither/tools/mutator/mutators/RR.py @@ -24,7 +24,7 @@ def _mutate(self) -> Dict: old_str = self.in_file_str[start:stop] line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: - if old_str != "revert()": + if not old_str.lstrip().startswith("revert"): new_str = "revert()" create_patch_with_line( result, From f15fbf1f3918f4fac3fc906389cfce4450c2f2c8 Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 7 Feb 2024 10:40:17 -0500 Subject: [PATCH 07/42] tweak logs --- .../tools/mutator/mutators/abstract_mutator.py | 1 - .../mutator/utils/testing_generated_mutant.py | 15 ++++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index 9df7c45ab..b3238c728 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -133,7 +133,6 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: self.total_mutant_counts[1] += 1 else: self.total_mutant_counts[2] += 1 - # logger.info(yellow(f"{self.VALID_MUTANTS_COUNT} valid mutants, {self.INVALID_MUTANTS_COUNT} invalid mutants")) return ( self.total_mutant_counts, diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 9780f1d9a..647c6988f 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -6,7 +6,7 @@ from typing import Dict import crytic_compile from slither.tools.mutator.utils.file_handling import create_mutant_file, reset_file -from slither.utils.colors import green, red, yellow +from slither.utils.colors import green, red #, yellow logger = logging.getLogger("Slither-Mutate") @@ -91,12 +91,13 @@ def test_patch( # pylint: disable=too-many-arguments reset_file(file) return 0 # valid else: - if verbose: - logger.info( - yellow( - f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> COMPILATION FAILURE" - ) - ) + # too noisy + # if verbose: + # logger.info( + # yellow( + # f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> COMPILATION FAILURE" + # ) + # ) reset_file(file) return 2 # compile failure From a3e6c4b4354f6cd82cac5ad6a505380bd2f5b35b Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 7 Feb 2024 11:22:44 -0500 Subject: [PATCH 08/42] reset mutant counts after each analyzed files --- slither/tools/mutator/__main__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 3324950a3..72331830e 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -243,6 +243,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t dont_mutate_lines = lines_list if not quick_flag: dont_mutate_lines = [] + except Exception as e: # pylint: disable=broad-except logger.error(e) @@ -271,4 +272,14 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t else: print(yellow("Zero Tweak mutants analyzed")) + # Reset mutant counts before moving on to the next file + total_mutant_counts[0] = 0 + total_mutant_counts[1] = 0 + total_mutant_counts[2] = 0 + valid_mutant_counts[0] = 0 + valid_mutant_counts[1] = 0 + valid_mutant_counts[2] = 0 + + print(magenta(f"Finished Mutation Campaign in '{args.codebase}' \n")) + # endregion From c990b9a9e9a934bfa4d1e6e7c2f8a62f03bd0d8b Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 7 Feb 2024 15:16:47 -0500 Subject: [PATCH 09/42] fix mutant count calculations & add more verbose logs --- slither/tools/mutator/__main__.py | 40 ++++++++++++------- .../mutator/mutators/abstract_mutator.py | 14 +++++++ .../mutator/utils/testing_generated_mutant.py | 19 ++++----- 3 files changed, 50 insertions(+), 23 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 72331830e..4b06db051 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -234,16 +234,36 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t dont_mutate_lines, ) (total_counts, valid_counts, lines_list) = m.mutate() - total_mutant_counts[0] += total_counts[0] - total_mutant_counts[1] += total_counts[1] - total_mutant_counts[2] += total_counts[2] - valid_mutant_counts[0] += valid_counts[0] - valid_mutant_counts[1] += valid_counts[1] - valid_mutant_counts[2] += valid_counts[2] + + logger.info(f"Mutator {m.NAME} has completed") + logger.info(f"Found {valid_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)") + logger.info(f"Found {valid_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)") + logger.info(f"Found {valid_counts[2]} uncaught tweak mutants (out of {total_counts[2]} that compile)") + logger.info("Setting these values ") + + if m.NAME == "RR": + total_mutant_counts[0] += total_counts[0] + valid_mutant_counts[0] += valid_counts[0] + elif m.NAME == "CR": + total_mutant_counts[1] += total_counts[1] + valid_mutant_counts[1] += valid_counts[1] + else: + total_mutant_counts[2] += total_counts[2] + valid_mutant_counts[2] += valid_counts[2] + dont_mutate_lines = lines_list if not quick_flag: dont_mutate_lines = [] + # Reset mutant counts before moving on to the next file + # TODO: is this logic in the right place..? + total_mutant_counts[0] = 0 + total_mutant_counts[1] = 0 + total_mutant_counts[2] = 0 + valid_mutant_counts[0] = 0 + valid_mutant_counts[1] = 0 + valid_mutant_counts[2] = 0 + except Exception as e: # pylint: disable=broad-except logger.error(e) @@ -272,14 +292,6 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t else: print(yellow("Zero Tweak mutants analyzed")) - # Reset mutant counts before moving on to the next file - total_mutant_counts[0] = 0 - total_mutant_counts[1] = 0 - total_mutant_counts[2] = 0 - valid_mutant_counts[0] = 0 - valid_mutant_counts[1] = 0 - valid_mutant_counts[2] = 0 - print(magenta(f"Finished Mutation Campaign in '{args.codebase}' \n")) # endregion diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index b3238c728..c761ac2ee 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -134,6 +134,20 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: else: self.total_mutant_counts[2] += 1 + if self.verbose: + if self.NAME == "RR": + logger.info(f"Found {self.valid_mutant_counts[0]} uncaught revert mutants so far (out of {self.total_mutant_counts[0]} that compile)") + elif self.NAME == "CR": + logger.info(f"Found {self.valid_mutant_counts[1]} uncaught comment mutants so far (out of {self.total_mutant_counts[1]} that compile)") + else: + logger.info(f"Found {self.valid_mutant_counts[2]} uncaught tweak mutants so far (out of {self.total_mutant_counts[2]} that compile)") + + if self.verbose: + logger.info(f"Done mutating file {file}") + logger.info(f"Found {self.valid_mutant_counts[0]} uncaught revert mutants (out of {self.total_mutant_counts[0]} that compile)") + logger.info(f"Found {self.valid_mutant_counts[1]} uncaught comment mutants (out of {self.total_mutant_counts[1]} that compile)") + logger.info(f"Found {self.valid_mutant_counts[2]} uncaught tweak mutants (out of {self.total_mutant_counts[2]} that compile)") + return ( self.total_mutant_counts, self.valid_mutant_counts, diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 647c6988f..df670aa79 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -6,7 +6,7 @@ from typing import Dict import crytic_compile from slither.tools.mutator.utils.file_handling import create_mutant_file, reset_file -from slither.utils.colors import green, red #, yellow +from slither.utils.colors import green, red, yellow logger = logging.getLogger("Slither-Mutate") @@ -58,7 +58,7 @@ def run_test_cmd(cmd: str, test_dir: str, timeout: int) -> bool: # if r is 0 then it is valid mutant because tests didn't fail return r == 0 - +# return 0 if valid, 1 if invalid, and 2 if compilation fails def test_patch( # pylint: disable=too-many-arguments file: str, patch: Dict, @@ -91,13 +91,13 @@ def test_patch( # pylint: disable=too-many-arguments reset_file(file) return 0 # valid else: - # too noisy - # if verbose: - # logger.info( - # yellow( - # f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> COMPILATION FAILURE" - # ) - # ) + if verbose: + logger.info( + yellow( + f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> COMPILATION FAILURE" + ) + ) + reset_file(file) return 2 # compile failure @@ -107,5 +107,6 @@ def test_patch( # pylint: disable=too-many-arguments f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> INVALID" ) ) + reset_file(file) return 1 # invalid From f23ca681d03dd579fce5f8e84e2e0d56586ab295 Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 7 Feb 2024 15:37:00 -0500 Subject: [PATCH 10/42] move mutant count reset logic --- slither/tools/mutator/__main__.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 4b06db051..3f385d64a 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -235,34 +235,32 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t ) (total_counts, valid_counts, lines_list) = m.mutate() - logger.info(f"Mutator {m.NAME} has completed") - logger.info(f"Found {valid_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)") - logger.info(f"Found {valid_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)") - logger.info(f"Found {valid_counts[2]} uncaught tweak mutants (out of {total_counts[2]} that compile)") - logger.info("Setting these values ") - if m.NAME == "RR": total_mutant_counts[0] += total_counts[0] valid_mutant_counts[0] += valid_counts[0] + logger.info(f"Mutator {m.NAME} found {valid_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)") elif m.NAME == "CR": total_mutant_counts[1] += total_counts[1] valid_mutant_counts[1] += valid_counts[1] + logger.info(f"Mutator {m.NAME} found {valid_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)") else: total_mutant_counts[2] += total_counts[2] valid_mutant_counts[2] += valid_counts[2] + logger.info(f"Mutator {m.NAME} found {valid_counts[2]} uncaught tweak mutants (out of {total_counts[2]} that compile)") dont_mutate_lines = lines_list if not quick_flag: dont_mutate_lines = [] - # Reset mutant counts before moving on to the next file - # TODO: is this logic in the right place..? - total_mutant_counts[0] = 0 - total_mutant_counts[1] = 0 - total_mutant_counts[2] = 0 - valid_mutant_counts[0] = 0 - valid_mutant_counts[1] = 0 - valid_mutant_counts[2] = 0 + # Reset mutant counts before moving on to the next file + # TODO: is this logic in the right place..? + logger.info("Reseting mutant counts to zero") + total_mutant_counts[0] = 0 + total_mutant_counts[1] = 0 + total_mutant_counts[2] = 0 + valid_mutant_counts[0] = 0 + valid_mutant_counts[1] = 0 + valid_mutant_counts[2] = 0 except Exception as e: # pylint: disable=broad-except logger.error(e) From 9ef4c2aeb9168bbe75129314f3c329153e30034f Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 7 Feb 2024 16:02:04 -0500 Subject: [PATCH 11/42] don't mutate interfaces --- slither/tools/mutator/__main__.py | 34 ++++++++++++++++++------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 3f385d64a..76d4df775 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -188,7 +188,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t mutators_list = CR_RR_list + mutators_list for filename in sol_file_list: # pylint: disable=too-many-nested-blocks - contract_name = os.path.split(filename)[1].split(".sol")[0] + file_name = os.path.split(filename)[1].split(".sol")[0] # slither object sl = Slither(filename, **vars(args)) # create a backup files @@ -201,33 +201,39 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t dont_mutate_lines = [] # mutation - contract_instance = '' + target_contract = "" try: for compilation_unit_of_main_file in sl.compilation_units: for contract in compilation_unit_of_main_file.contracts: if contract_names is not None and contract.name in contract_names: - contract_instance = contract + target_contract = contract elif contract_names is not None and contract.name not in contract_names: - contract_instance = "SLITHER_SKIP_MUTATIONS" - elif str(contract.name).lower() == contract_name.lower(): - contract_instance = contract - - if contract_instance == '': - logger.info(f"Cannot find contracts in file {filename}, try specifying them with --contract-names") + target_contract = "SLITHER_SKIP_MUTATIONS" + elif contract.name.lower() == file_name.lower(): + target_contract = contract + if target_contract == "": + logger.info( + f"Cannot find contracts in file {filename}, try specifying them with --contract-names" + ) continue - - if contract_instance == 'SLITHER_SKIP_MUTATIONS': + if target_contract == "SLITHER_SKIP_MUTATIONS": logger.debug(f"Skipping mutations in {filename}") continue - logger.info(yellow(f"Mutating contract {contract_instance}")) + # TODO: find a more specific way to omit interfaces + # Ideally, we wouldn't depend on naming conventions + if target_contract.name.startswith("I"): + logger.debug(f"Skipping mutations on interface {filename}") + continue + + logger.info(yellow(f"Mutating contract {target_contract}")) for M in mutators_list: m = M( compilation_unit_of_main_file, int(timeout), test_command, test_directory, - contract_instance, + target_contract, solc_remappings, verbose, output_folder, @@ -270,7 +276,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t logger.error("\nExecution interrupted by user (Ctrl + C). Cleaning up...") transfer_and_delete(files_dict) - if not contract_instance == 'SLITHER_SKIP_MUTATIONS': + if not target_contract == 'SLITHER_SKIP_MUTATIONS': # transfer and delete the backup files transfer_and_delete(files_dict) # output From fcbd3278423f1b858b42fa0418cdcd67d0435682 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 12 Feb 2024 10:52:21 -0500 Subject: [PATCH 12/42] remove redundant skip-mutating-contract logic --- slither/tools/mutator/__main__.py | 49 +++++++++++++++++++------------ 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 76d4df775..3a6956819 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -276,25 +276,36 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t logger.error("\nExecution interrupted by user (Ctrl + C). Cleaning up...") transfer_and_delete(files_dict) - if not target_contract == 'SLITHER_SKIP_MUTATIONS': - # transfer and delete the backup files - transfer_and_delete(files_dict) - # output - print(yellow(f"Done mutating {filename}.")) - if total_mutant_counts[0] > 0: - print(yellow(f"Revert mutants: {valid_mutant_counts[0]} valid of {total_mutant_counts[0]} ({100 * valid_mutant_counts[0]/total_mutant_counts[0]}%)")) - else: - print(yellow("Zero Revert mutants analyzed")) - - if total_mutant_counts[1] > 0: - print(yellow(f"Comment mutants: {valid_mutant_counts[1]} valid of {total_mutant_counts[1]} ({100 * valid_mutant_counts[1]/total_mutant_counts[1]}%)")) - else: - print(yellow("Zero Comment mutants analyzed")) - - if total_mutant_counts[2] > 0: - print(yellow(f"Tweak mutants: {valid_mutant_counts[2]} valid of {total_mutant_counts[2]} ({100 * valid_mutant_counts[2]/total_mutant_counts[2]}%)")) - else: - print(yellow("Zero Tweak mutants analyzed")) + # transfer and delete the backup files + transfer_and_delete(files_dict) + # output + print(yellow(f"Done mutating {filename}.")) + if total_mutant_counts[0] > 0: + print( + yellow( + f"Revert mutants: {valid_mutant_counts[0]} valid of {total_mutant_counts[0]} ({100 * valid_mutant_counts[0]/total_mutant_counts[0]}%)" + ) + ) + else: + print(yellow("Zero Revert mutants analyzed")) + + if total_mutant_counts[1] > 0: + print( + yellow( + f"Comment mutants: {valid_mutant_counts[1]} valid of {total_mutant_counts[1]} ({100 * valid_mutant_counts[1]/total_mutant_counts[1]}%)" + ) + ) + else: + print(yellow("Zero Comment mutants analyzed")) + + if total_mutant_counts[2] > 0: + print( + yellow( + f"Tweak mutants: {valid_mutant_counts[2]} valid of {total_mutant_counts[2]} ({100 * valid_mutant_counts[2]/total_mutant_counts[2]}%)" + ) + ) + else: + print(yellow("Zero Tweak mutants analyzed")) print(magenta(f"Finished Mutation Campaign in '{args.codebase}' \n")) From 5d43f9e28ed7ddf0a01d9b41a694dce268ca7eb4 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 12 Feb 2024 11:16:45 -0500 Subject: [PATCH 13/42] use dedicated variable for naming mutant files --- slither/tools/mutator/mutators/abstract_mutator.py | 7 ++----- slither/tools/mutator/utils/file_handling.py | 12 ++++++++++-- .../tools/mutator/utils/testing_generated_mutant.py | 3 +-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index c761ac2ee..aaeffc49a 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -19,7 +19,6 @@ class AbstractMutator( ): # pylint: disable=too-few-public-methods,too-many-instance-attributes NAME = "" HELP = "" - INVALID_MUTANTS_COUNT = 0 VALID_MUTANTS_COUNT = 0 VALID_RR_MUTANTS_COUNT = 0 VALID_CR_MUTANTS_COUNT = 0 @@ -82,9 +81,8 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: (all_patches) = self._mutate() if "patches" not in all_patches: logger.debug("No patches found by %s", self.NAME) - return ([0,0,0], [0,0,0], self.dont_mutate_line) - - for file in all_patches["patches"]: + return ([0, 0, 0], [0, 0, 0], self.dont_mutate_line) + for file in all_patches["patches"]: # Note: This should only loop over a single file original_txt = self.slither.source_code[file].encode("utf8") patches = all_patches["patches"][file] patches.sort(key=lambda x: x["start"]) @@ -96,7 +94,6 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: file, patch, self.test_command, - self.VALID_MUTANTS_COUNT, self.NAME, self.timeout, self.solc_remappings, diff --git a/slither/tools/mutator/utils/file_handling.py b/slither/tools/mutator/utils/file_handling.py index ddb3efb50..08eb8f51c 100644 --- a/slither/tools/mutator/utils/file_handling.py +++ b/slither/tools/mutator/utils/file_handling.py @@ -1,4 +1,5 @@ import os +import traceback from typing import Dict, List import logging @@ -46,9 +47,13 @@ def transfer_and_delete(files_dict: Dict) -> None: logger.error(f"Error transferring content: {e}") -def create_mutant_file(file: str, count: int, rule: str) -> None: +global_counter = {} + +def create_mutant_file(file: str, rule: str) -> None: """function to create new mutant file""" try: + if rule not in global_counter: + global_counter[rule] = 0 _, filename = os.path.split(file) # Read content from the duplicated file with open(file, "r", encoding="utf8") as source_file: @@ -67,12 +72,13 @@ def create_mutant_file(file: str, count: int, rule: str) -> None: + "_" + rule + "_" - + str(count) + + str(global_counter[rule]) + ".sol", "w", encoding="utf8", ) as mutant_file: mutant_file.write(content) + global_counter[rule] += 1 # reset the file with open(duplicated_files[file], "r", encoding="utf8") as duplicated_file: @@ -83,6 +89,8 @@ def create_mutant_file(file: str, count: int, rule: str) -> None: except Exception as e: # pylint: disable=broad-except logger.error(f"Error creating mutant: {e}") + traceback_str = traceback.format_exc() + logger.error(traceback_str) # Log the stack trace def reset_file(file: str) -> None: diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index df670aa79..15ae010dc 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -63,7 +63,6 @@ def test_patch( # pylint: disable=too-many-arguments file: str, patch: Dict, command: str, - index: int, generator_name: str, timeout: int, mappings: str | None, @@ -82,7 +81,7 @@ def test_patch( # pylint: disable=too-many-arguments filepath.write(replaced_content) if compile_generated_mutant(file, mappings): if run_test_cmd(command, file, timeout): - create_mutant_file(file, index, generator_name) + create_mutant_file(file, generator_name) logger.info( red( f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> VALID" From 69748c8aa3e378f64fed4523be1170090da82f85 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 12 Feb 2024 14:57:48 -0500 Subject: [PATCH 14/42] update debug logs and mutant counters --- slither/tools/mutator/__main__.py | 3 +++ slither/tools/mutator/mutators/abstract_mutator.py | 11 ++++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 3a6956819..38f23cc3e 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -245,14 +245,17 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t total_mutant_counts[0] += total_counts[0] valid_mutant_counts[0] += valid_counts[0] logger.info(f"Mutator {m.NAME} found {valid_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)") + logger.info(f"Running total: found {valid_mutant_counts[0]} uncaught revert mutants (out of {total_mutant_counts[0]} that compile)") elif m.NAME == "CR": total_mutant_counts[1] += total_counts[1] valid_mutant_counts[1] += valid_counts[1] logger.info(f"Mutator {m.NAME} found {valid_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)") + logger.info(f"Running total: found {valid_mutant_counts[1]} uncaught comment mutants (out of {total_mutant_counts[1]} that compile)") else: total_mutant_counts[2] += total_counts[2] valid_mutant_counts[2] += valid_counts[2] logger.info(f"Mutator {m.NAME} found {valid_counts[2]} uncaught tweak mutants (out of {total_counts[2]} that compile)") + logger.info(f"Running total: found {valid_mutant_counts[2]} uncaught tweak mutants (out of {total_mutant_counts[2]} that compile)") dont_mutate_lines = lines_list if not quick_flag: diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index aaeffc49a..e16b1d898 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -19,13 +19,6 @@ class AbstractMutator( ): # pylint: disable=too-few-public-methods,too-many-instance-attributes NAME = "" HELP = "" - VALID_MUTANTS_COUNT = 0 - VALID_RR_MUTANTS_COUNT = 0 - VALID_CR_MUTANTS_COUNT = 0 - # total revert/comment/tweak mutants that were generated and compiled - total_mutant_counts = [0, 0, 0] - # total valid revert/comment/tweak mutants - valid_mutant_counts = [0, 0, 0] def __init__( # pylint: disable=too-many-arguments self, @@ -55,6 +48,10 @@ def __init__( # pylint: disable=too-many-arguments self.in_file = self.contract.source_mapping.filename.absolute self.in_file_str = self.contract.compilation_unit.core.source_code[self.in_file] self.dont_mutate_line = dont_mutate_line + # total revert/comment/tweak mutants that were generated and compiled + self.total_mutant_counts = [0, 0, 0] + # total valid revert/comment/tweak mutants + self.valid_mutant_counts = [0, 0, 0] if not self.NAME: raise IncorrectMutatorInitialization( From c7cd1378aca46027bab6ad6a3c20e0f2b8a1b519 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 12 Feb 2024 15:29:08 -0500 Subject: [PATCH 15/42] hide very verbose logs behind a -vv flag --- slither/tools/mutator/__main__.py | 35 ++++++++++++------- .../mutator/mutators/abstract_mutator.py | 5 ++- .../mutator/utils/testing_generated_mutant.py | 5 +-- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 38f23cc3e..73bcba76e 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -67,8 +67,18 @@ def parse_args() -> argparse.Namespace: # to print just all the mutants parser.add_argument( + "-v", "--verbose", - help="output all mutants generated", + help="log mutants that are caught as well as those that are uncaught", + action="store_true", + default=False, + ) + + # to print just all the mutants + parser.add_argument( + "-vv", + "--very-verbose", + help="log mutants that are caught, uncaught, and fail to compile. And more!", action="store_true", default=False, ) @@ -146,6 +156,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t timeout: Optional[int] = args.timeout solc_remappings: Optional[str] = args.solc_remaps verbose: Optional[bool] = args.verbose + very_verbose: Optional[bool] = args.very_verbose mutators_to_run: Optional[List[str]] = args.mutators_to_run contract_names: Optional[List[str]] = args.contract_names quick_flag: Optional[bool] = args.quick @@ -236,6 +247,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t target_contract, solc_remappings, verbose, + very_verbose, output_folder, dont_mutate_lines, ) @@ -245,12 +257,10 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t total_mutant_counts[0] += total_counts[0] valid_mutant_counts[0] += valid_counts[0] logger.info(f"Mutator {m.NAME} found {valid_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)") - logger.info(f"Running total: found {valid_mutant_counts[0]} uncaught revert mutants (out of {total_mutant_counts[0]} that compile)") elif m.NAME == "CR": total_mutant_counts[1] += total_counts[1] valid_mutant_counts[1] += valid_counts[1] logger.info(f"Mutator {m.NAME} found {valid_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)") - logger.info(f"Running total: found {valid_mutant_counts[1]} uncaught comment mutants (out of {total_mutant_counts[1]} that compile)") else: total_mutant_counts[2] += total_counts[2] valid_mutant_counts[2] += valid_counts[2] @@ -261,16 +271,6 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t if not quick_flag: dont_mutate_lines = [] - # Reset mutant counts before moving on to the next file - # TODO: is this logic in the right place..? - logger.info("Reseting mutant counts to zero") - total_mutant_counts[0] = 0 - total_mutant_counts[1] = 0 - total_mutant_counts[2] = 0 - valid_mutant_counts[0] = 0 - valid_mutant_counts[1] = 0 - valid_mutant_counts[2] = 0 - except Exception as e: # pylint: disable=broad-except logger.error(e) @@ -310,6 +310,15 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t else: print(yellow("Zero Tweak mutants analyzed")) + # Reset mutant counts before moving on to the next file + logger.info("Reseting mutant counts to zero") + total_mutant_counts[0] = 0 + total_mutant_counts[1] = 0 + total_mutant_counts[2] = 0 + valid_mutant_counts[0] = 0 + valid_mutant_counts[1] = 0 + valid_mutant_counts[2] = 0 + print(magenta(f"Finished Mutation Campaign in '{args.codebase}' \n")) # endregion diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index e16b1d898..f44f5004b 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -29,6 +29,7 @@ def __init__( # pylint: disable=too-many-arguments contract_instance: Contract, solc_remappings: str | None, verbose: bool, + very_verbose: bool, output_folder: str, dont_mutate_line: List[int], rate: int = 10, @@ -43,6 +44,7 @@ def __init__( # pylint: disable=too-many-arguments self.timeout = timeout self.solc_remappings = solc_remappings self.verbose = verbose + self.very_verbose = very_verbose self.output_folder = output_folder self.contract = contract_instance self.in_file = self.contract.source_mapping.filename.absolute @@ -95,6 +97,7 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: self.timeout, self.solc_remappings, self.verbose, + self.very_verbose, ) # count the valid mutants, flag RR/CR mutants to skip further mutations @@ -128,7 +131,7 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: else: self.total_mutant_counts[2] += 1 - if self.verbose: + if self.very_verbose: if self.NAME == "RR": logger.info(f"Found {self.valid_mutant_counts[0]} uncaught revert mutants so far (out of {self.total_mutant_counts[0]} that compile)") elif self.NAME == "CR": diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 15ae010dc..7e6602d83 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -67,7 +67,8 @@ def test_patch( # pylint: disable=too-many-arguments timeout: int, mappings: str | None, verbose: bool, -) -> bool: + very_verbose: bool, +) -> int: """ function to verify the validity of each patch returns: valid or invalid patch @@ -90,7 +91,7 @@ def test_patch( # pylint: disable=too-many-arguments reset_file(file) return 0 # valid else: - if verbose: + if very_verbose: logger.info( yellow( f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> COMPILATION FAILURE" From fa8be857cc11fa3e2aeacfb9f2b59e528bdfadd7 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 12 Feb 2024 16:00:14 -0500 Subject: [PATCH 16/42] tidy up logs --- slither/tools/mutator/__main__.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 73bcba76e..dbcb528df 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -237,7 +237,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t logger.debug(f"Skipping mutations on interface {filename}") continue - logger.info(yellow(f"Mutating contract {target_contract}")) + logger.info(yellow(f"\nMutating contract {target_contract}")) for M in mutators_list: m = M( compilation_unit_of_main_file, @@ -256,16 +256,16 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t if m.NAME == "RR": total_mutant_counts[0] += total_counts[0] valid_mutant_counts[0] += valid_counts[0] - logger.info(f"Mutator {m.NAME} found {valid_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)") + logger.info(yellow(f"Mutator {m.NAME} found {valid_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)")) elif m.NAME == "CR": total_mutant_counts[1] += total_counts[1] valid_mutant_counts[1] += valid_counts[1] - logger.info(f"Mutator {m.NAME} found {valid_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)") + logger.info(yellow(f"Mutator {m.NAME} found {valid_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)")) else: total_mutant_counts[2] += total_counts[2] valid_mutant_counts[2] += valid_counts[2] - logger.info(f"Mutator {m.NAME} found {valid_counts[2]} uncaught tweak mutants (out of {total_counts[2]} that compile)") - logger.info(f"Running total: found {valid_mutant_counts[2]} uncaught tweak mutants (out of {total_mutant_counts[2]} that compile)") + logger.info(yellow(f"Mutator {m.NAME} found {valid_counts[2]} uncaught tweak mutants (out of {total_counts[2]} that compile)")) + logger.info(yellow(f"Running total: found {valid_mutant_counts[2]} uncaught tweak mutants (out of {total_mutant_counts[2]} that compile)")) dont_mutate_lines = lines_list if not quick_flag: @@ -282,36 +282,37 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t # transfer and delete the backup files transfer_and_delete(files_dict) # output - print(yellow(f"Done mutating {filename}.")) + logger.info(yellow(f"Done mutating {target_contract}.")) if total_mutant_counts[0] > 0: - print( + logger.info( yellow( f"Revert mutants: {valid_mutant_counts[0]} valid of {total_mutant_counts[0]} ({100 * valid_mutant_counts[0]/total_mutant_counts[0]}%)" ) ) else: - print(yellow("Zero Revert mutants analyzed")) + logger.info(yellow("Zero Revert mutants analyzed")) if total_mutant_counts[1] > 0: - print( + logger.info( yellow( f"Comment mutants: {valid_mutant_counts[1]} valid of {total_mutant_counts[1]} ({100 * valid_mutant_counts[1]/total_mutant_counts[1]}%)" ) ) else: - print(yellow("Zero Comment mutants analyzed")) + logger.info(yellow("Zero Comment mutants analyzed")) if total_mutant_counts[2] > 0: - print( + logger.info( yellow( f"Tweak mutants: {valid_mutant_counts[2]} valid of {total_mutant_counts[2]} ({100 * valid_mutant_counts[2]/total_mutant_counts[2]}%)" ) ) else: - print(yellow("Zero Tweak mutants analyzed")) + logger.info(yellow("Zero Tweak mutants analyzed")) # Reset mutant counts before moving on to the next file - logger.info("Reseting mutant counts to zero") + if very_verbose: + logger.info(yellow("Reseting mutant counts to zero")) total_mutant_counts[0] = 0 total_mutant_counts[1] = 0 total_mutant_counts[2] = 0 @@ -319,6 +320,6 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t valid_mutant_counts[1] = 0 valid_mutant_counts[2] = 0 - print(magenta(f"Finished Mutation Campaign in '{args.codebase}' \n")) + logger.info(magenta(f"Finished Mutation Campaign in '{args.codebase}' \n")) # endregion From ef0e354f0d3ed020b1f126ac7f4feec8f54c6f4e Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 12 Feb 2024 16:15:01 -0500 Subject: [PATCH 17/42] rename in/valid mutants to un/caught --- slither/tools/mutator/__main__.py | 66 ++++++++++--------- .../mutator/mutators/abstract_mutator.py | 45 ++++++------- .../mutator/utils/testing_generated_mutant.py | 16 ++--- 3 files changed, 62 insertions(+), 65 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index dbcb528df..8047d347a 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -8,7 +8,7 @@ from crytic_compile import cryticparser from slither import Slither from slither.tools.mutator.mutators import all_mutators -from slither.utils.colors import yellow, magenta +from slither.utils.colors import blue, magenta from .mutators.abstract_mutator import AbstractMutator from .utils.command_line import output_mutators from .utils.file_handling import ( @@ -161,18 +161,18 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t contract_names: Optional[List[str]] = args.contract_names quick_flag: Optional[bool] = args.quick - logger.info(magenta(f"Starting Mutation Campaign in '{args.codebase} \n")) + logger.info(blue(f"Starting Mutation Campaign in '{args.codebase}")) if paths_to_ignore: paths_to_ignore_list = paths_to_ignore.strip("][").split(",") - logger.info(magenta(f"Ignored paths - {', '.join(paths_to_ignore_list)} \n")) + logger.info(blue(f"Ignored paths - {', '.join(paths_to_ignore_list)}")) else: paths_to_ignore_list = [] # get all the contracts as a list from given codebase sol_file_list: List[str] = get_sol_file_list(args.codebase, paths_to_ignore_list) - # folder where backup files and valid mutants created + # folder where backup files and uncaught mutants are saved if output_dir is None: output_dir = "/mutation_campaign" output_folder = os.getcwd() + output_dir @@ -206,8 +206,8 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t files_dict = backup_source_file(sl.source_code, output_folder) # total revert/comment/tweak mutants that were generated and compiled total_mutant_counts = [0, 0, 0] - # total valid revert/comment/tweak mutants - valid_mutant_counts = [0, 0, 0] + # total uncaught revert/comment/tweak mutants + uncaught_mutant_counts = [0, 0, 0] # lines those need not be mutated (taken from RR and CR) dont_mutate_lines = [] @@ -237,7 +237,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t logger.debug(f"Skipping mutations on interface {filename}") continue - logger.info(yellow(f"\nMutating contract {target_contract}")) + logger.info(blue(f"Mutating contract {target_contract}")) for M in mutators_list: m = M( compilation_unit_of_main_file, @@ -251,21 +251,24 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t output_folder, dont_mutate_lines, ) - (total_counts, valid_counts, lines_list) = m.mutate() + (total_counts, uncaught_counts, lines_list) = m.mutate() if m.NAME == "RR": total_mutant_counts[0] += total_counts[0] - valid_mutant_counts[0] += valid_counts[0] - logger.info(yellow(f"Mutator {m.NAME} found {valid_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)")) + uncaught_mutant_counts[0] += uncaught_counts[0] + if verbose: + logger.info(magenta(f"Mutator {m.NAME} found {uncaught_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)")) elif m.NAME == "CR": total_mutant_counts[1] += total_counts[1] - valid_mutant_counts[1] += valid_counts[1] - logger.info(yellow(f"Mutator {m.NAME} found {valid_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)")) + uncaught_mutant_counts[1] += uncaught_counts[1] + if verbose: + logger.info(magenta(f"Mutator {m.NAME} found {uncaught_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)")) else: total_mutant_counts[2] += total_counts[2] - valid_mutant_counts[2] += valid_counts[2] - logger.info(yellow(f"Mutator {m.NAME} found {valid_counts[2]} uncaught tweak mutants (out of {total_counts[2]} that compile)")) - logger.info(yellow(f"Running total: found {valid_mutant_counts[2]} uncaught tweak mutants (out of {total_mutant_counts[2]} that compile)")) + uncaught_mutant_counts[2] += uncaught_counts[2] + if verbose: + logger.info(magenta(f"Mutator {m.NAME} found {uncaught_counts[2]} uncaught tweak mutants (out of {total_counts[2]} that compile)")) + logger.info(magenta(f"Running total: found {uncaught_mutant_counts[2]} uncaught tweak mutants (out of {total_mutant_counts[2]} that compile)")) dont_mutate_lines = lines_list if not quick_flag: @@ -281,45 +284,46 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t # transfer and delete the backup files transfer_and_delete(files_dict) - # output - logger.info(yellow(f"Done mutating {target_contract}.")) + + # log results for this file + logger.info(blue(f"Done mutating {target_contract}.")) if total_mutant_counts[0] > 0: logger.info( - yellow( - f"Revert mutants: {valid_mutant_counts[0]} valid of {total_mutant_counts[0]} ({100 * valid_mutant_counts[0]/total_mutant_counts[0]}%)" + magenta( + f"Revert mutants: {uncaught_mutant_counts[0]} uncaught of {total_mutant_counts[0]} ({100 * uncaught_mutant_counts[0]/total_mutant_counts[0]}%)" ) ) else: - logger.info(yellow("Zero Revert mutants analyzed")) + logger.info(magenta("Zero Revert mutants analyzed")) if total_mutant_counts[1] > 0: logger.info( - yellow( - f"Comment mutants: {valid_mutant_counts[1]} valid of {total_mutant_counts[1]} ({100 * valid_mutant_counts[1]/total_mutant_counts[1]}%)" + magenta( + f"Comment mutants: {uncaught_mutant_counts[1]} uncaught of {total_mutant_counts[1]} ({100 * uncaught_mutant_counts[1]/total_mutant_counts[1]}%)" ) ) else: - logger.info(yellow("Zero Comment mutants analyzed")) + logger.info(magenta("Zero Comment mutants analyzed")) if total_mutant_counts[2] > 0: logger.info( - yellow( - f"Tweak mutants: {valid_mutant_counts[2]} valid of {total_mutant_counts[2]} ({100 * valid_mutant_counts[2]/total_mutant_counts[2]}%)" + magenta( + f"Tweak mutants: {uncaught_mutant_counts[2]} uncaught of {total_mutant_counts[2]} ({100 * uncaught_mutant_counts[2]/total_mutant_counts[2]}%)\n" ) ) else: - logger.info(yellow("Zero Tweak mutants analyzed")) + logger.info(magenta("Zero Tweak mutants analyzed\n")) # Reset mutant counts before moving on to the next file if very_verbose: - logger.info(yellow("Reseting mutant counts to zero")) + logger.info(blue("Reseting mutant counts to zero")) total_mutant_counts[0] = 0 total_mutant_counts[1] = 0 total_mutant_counts[2] = 0 - valid_mutant_counts[0] = 0 - valid_mutant_counts[1] = 0 - valid_mutant_counts[2] = 0 + uncaught_mutant_counts[0] = 0 + uncaught_mutant_counts[1] = 0 + uncaught_mutant_counts[2] = 0 - logger.info(magenta(f"Finished Mutation Campaign in '{args.codebase}' \n")) + logger.info(blue(f"Finished Mutation Campaign in '{args.codebase}' \n")) # endregion diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index f44f5004b..80665d1dc 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -52,8 +52,8 @@ def __init__( # pylint: disable=too-many-arguments self.dont_mutate_line = dont_mutate_line # total revert/comment/tweak mutants that were generated and compiled self.total_mutant_counts = [0, 0, 0] - # total valid revert/comment/tweak mutants - self.valid_mutant_counts = [0, 0, 0] + # total uncaught revert/comment/tweak mutants + self.uncaught_mutant_counts = [0, 0, 0] if not self.NAME: raise IncorrectMutatorInitialization( @@ -88,8 +88,7 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: logger.info(yellow(f"Mutating {file} with {self.NAME} \n")) for patch in patches: # test the patch - - patchIsValid = test_patch( + patchWasCaught = test_patch( file, patch, self.test_command, @@ -100,16 +99,16 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: self.very_verbose, ) - # count the valid mutants, flag RR/CR mutants to skip further mutations - if patchIsValid == 0: - if self.NAME == 'RR': - self.valid_mutant_counts[0] += 1 - self.dont_mutate_line.append(patch['line_number']) - elif self.NAME == 'CR': - self.valid_mutant_counts[1] += 1 - self.dont_mutate_line.append(patch['line_number']) + # count the uncaught mutants, flag RR/CR mutants to skip further mutations + if patchWasCaught == 0: + if self.NAME == "RR": + self.uncaught_mutant_counts[0] += 1 + self.dont_mutate_line.append(patch["line_number"]) + elif self.NAME == "CR": + self.uncaught_mutant_counts[1] += 1 + self.dont_mutate_line.append(patch["line_number"]) else: - self.valid_mutant_counts[2] += 1 + self.uncaught_mutant_counts[2] += 1 patched_txt,_ = apply_patch(original_txt, patch, 0) diff = create_diff(self.compilation_unit, original_txt, patched_txt, file) @@ -121,10 +120,10 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: self.output_folder + "/patches_file.txt", "a", encoding="utf8" ) as patches_file: patches_file.write(diff + "\n") - # count the total number of mutants that we were able to compile - if patchIsValid != 2: - if self.NAME == 'RR': + if patchWasCaught != 2: + if self.NAME == "RR": + self.total_mutant_counts[0] += 1 elif self.NAME == 'CR': self.total_mutant_counts[1] += 1 @@ -133,20 +132,14 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: if self.very_verbose: if self.NAME == "RR": - logger.info(f"Found {self.valid_mutant_counts[0]} uncaught revert mutants so far (out of {self.total_mutant_counts[0]} that compile)") + logger.info(f"Found {self.uncaught_mutant_counts[0]} uncaught revert mutants so far (out of {self.total_mutant_counts[0]} that compile)") elif self.NAME == "CR": - logger.info(f"Found {self.valid_mutant_counts[1]} uncaught comment mutants so far (out of {self.total_mutant_counts[1]} that compile)") + logger.info(f"Found {self.uncaught_mutant_counts[1]} uncaught comment mutants so far (out of {self.total_mutant_counts[1]} that compile)") else: - logger.info(f"Found {self.valid_mutant_counts[2]} uncaught tweak mutants so far (out of {self.total_mutant_counts[2]} that compile)") - - if self.verbose: - logger.info(f"Done mutating file {file}") - logger.info(f"Found {self.valid_mutant_counts[0]} uncaught revert mutants (out of {self.total_mutant_counts[0]} that compile)") - logger.info(f"Found {self.valid_mutant_counts[1]} uncaught comment mutants (out of {self.total_mutant_counts[1]} that compile)") - logger.info(f"Found {self.valid_mutant_counts[2]} uncaught tweak mutants (out of {self.total_mutant_counts[2]} that compile)") + logger.info(f"Found {self.uncaught_mutant_counts[2]} uncaught tweak mutants so far (out of {self.total_mutant_counts[2]} that compile)") return ( self.total_mutant_counts, - self.valid_mutant_counts, + self.uncaught_mutant_counts, self.dont_mutate_line ) diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 7e6602d83..78dfd5bcc 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -55,10 +55,10 @@ def run_test_cmd(cmd: str, test_dir: str, timeout: int) -> bool: # indicates whether the command executed sucessfully or not r = P.returncode - # if r is 0 then it is valid mutant because tests didn't fail + # if result is 0 then it is an uncaught mutant because tests didn't fail return r == 0 -# return 0 if valid, 1 if invalid, and 2 if compilation fails +# return 0 if uncaught, 1 if caught, and 2 if compilation fails def test_patch( # pylint: disable=too-many-arguments file: str, patch: Dict, @@ -70,8 +70,8 @@ def test_patch( # pylint: disable=too-many-arguments very_verbose: bool, ) -> int: """ - function to verify the validity of each patch - returns: valid or invalid patch + function to verify whether each patch is caught by tests + returns: 0 (uncaught), 1 (caught), or 2 (compilation failure) """ with open(file, "r", encoding="utf-8") as filepath: content = filepath.read() @@ -85,11 +85,11 @@ def test_patch( # pylint: disable=too-many-arguments create_mutant_file(file, generator_name) logger.info( red( - f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> VALID" + f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> UNCAUGHT" ) ) reset_file(file) - return 0 # valid + return 0 # uncaught else: if very_verbose: logger.info( @@ -104,9 +104,9 @@ def test_patch( # pylint: disable=too-many-arguments if verbose: logger.info( green( - f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> INVALID" + f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> CAUGHT" ) ) reset_file(file) - return 1 # invalid + return 1 # caught From 43ec72f78398f84527539c065f50851a9f91812a Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 08:46:46 -0500 Subject: [PATCH 18/42] log warning instead of crashing on test timeout/interruption --- .../mutator/utils/testing_generated_mutant.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 78dfd5bcc..ea8fce54e 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -1,12 +1,12 @@ -import subprocess -import os import logging +# import os +# import signal +import subprocess import time -import signal from typing import Dict import crytic_compile from slither.tools.mutator.utils.file_handling import create_mutant_file, reset_file -from slither.utils.colors import green, red, yellow +from slither.utils.colors import green, red logger = logging.getLogger("Slither-Mutate") @@ -47,11 +47,13 @@ def run_test_cmd(cmd: str, test_dir: str, timeout: int) -> bool: time.sleep(0.05) finally: if P.poll() is None: - logger.error("HAD TO TERMINATE ANALYSIS (TIMEOUT OR EXCEPTION)") - # sends a SIGTERM signal to process group - bascially killing the process - os.killpg(os.getpgid(P.pid), signal.SIGTERM) - # Avoid any weird race conditions from grabbing the return code - time.sleep(0.05) + # Timeout, treat this as a test failure + logger.error(f"Tests took too long, consider increasing the timeout value of {timeout}") + r = 1 + # # sends a SIGTERM signal to process group - bascially killing the process + # os.killpg(os.getpgid(P.pid), signal.SIGTERM) + # # Avoid any weird race conditions from grabbing the return code + # time.sleep(0.05) # indicates whether the command executed sucessfully or not r = P.returncode From 9862a1b1136d167fd90b34734640388b33c076d8 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 09:29:09 -0500 Subject: [PATCH 19/42] exit testing subprocess more gracefully on ctrl-c or timeout --- .../mutator/utils/testing_generated_mutant.py | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index ea8fce54e..1be367c81 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -1,8 +1,6 @@ import logging -# import os -# import signal +import sys import subprocess -import time from typing import Dict import crytic_compile from slither.tools.mutator.utils.file_handling import create_mutant_file, reset_file @@ -37,28 +35,31 @@ def run_test_cmd(cmd: str, test_dir: str, timeout: int) -> bool: elif "hardhat test" in cmd or "truffle test" in cmd and "--bail" not in cmd: cmd += " --bail" - start = time.time() + try: + result = subprocess.run( + cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False # True: Raises a CalledProcessError if the return code is non-zero + ) - # starting new process - with subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as P: - try: - # checking whether the process is completed or not within 30 seconds(default) - while P.poll() is None and (time.time() - start) < timeout: - time.sleep(0.05) - finally: - if P.poll() is None: - # Timeout, treat this as a test failure - logger.error(f"Tests took too long, consider increasing the timeout value of {timeout}") - r = 1 - # # sends a SIGTERM signal to process group - bascially killing the process - # os.killpg(os.getpgid(P.pid), signal.SIGTERM) - # # Avoid any weird race conditions from grabbing the return code - # time.sleep(0.05) - # indicates whether the command executed sucessfully or not - r = P.returncode + except subprocess.TimeoutExpired: + # Timeout, treat this as a test failure + logger.info("Tests took too long, consider increasing the timeout") + result = None # or set result to a default value + + except KeyboardInterrupt: + logger.info("Ctrl-C received. Exiting.") + sys.exit(1) # if result is 0 then it is an uncaught mutant because tests didn't fail - return r == 0 + if result: + code = result.returncode + return code == 0 + + return False # return 0 if uncaught, 1 if caught, and 2 if compilation fails def test_patch( # pylint: disable=too-many-arguments From 15e56eea71b7f71042ef2f0ffde513ebbdb844b2 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 10:02:47 -0500 Subject: [PATCH 20/42] clean up mutated files on interrupt --- .../mutator/utils/testing_generated_mutant.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 1be367c81..3cef50375 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -4,7 +4,7 @@ from typing import Dict import crytic_compile from slither.tools.mutator.utils.file_handling import create_mutant_file, reset_file -from slither.utils.colors import green, red +from slither.utils.colors import green, red, yellow logger = logging.getLogger("Slither-Mutate") @@ -21,13 +21,12 @@ def compile_generated_mutant(file_path: str, mappings: str) -> bool: return False -def run_test_cmd(cmd: str, test_dir: str, timeout: int) -> bool: +def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None) -> bool: """ function to run codebase tests returns: boolean whether the tests passed or not """ - # future purpose - _ = test_dir + # add --fail-fast for foundry tests, to exit after first failure if "forge test" in cmd and "--fail-fast" not in cmd: cmd += " --fail-fast" @@ -51,7 +50,11 @@ def run_test_cmd(cmd: str, test_dir: str, timeout: int) -> bool: result = None # or set result to a default value except KeyboardInterrupt: - logger.info("Ctrl-C received. Exiting.") + logger.info("Ctrl-C received") + if target_file is not None: + logger.info("Restoring original files") + reset_file(target_file) + logger.info("Exiting") sys.exit(1) # if result is 0 then it is an uncaught mutant because tests didn't fail @@ -84,7 +87,7 @@ def test_patch( # pylint: disable=too-many-arguments with open(file, "w", encoding="utf-8") as filepath: filepath.write(replaced_content) if compile_generated_mutant(file, mappings): - if run_test_cmd(command, file, timeout): + if run_test_cmd(command, timeout, file): create_mutant_file(file, generator_name) logger.info( red( From dcea61a3c933415adc6bdaa1937feef0b13ec92e Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 13:35:08 -0500 Subject: [PATCH 21/42] ensure timeout is an int --- slither/tools/mutator/__main__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 8047d347a..6147138ec 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -182,6 +182,8 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t # set default timeout if timeout is None: timeout = 30 + else: + timeout = int(timeout) # setting RR mutator as first mutator mutators_list = _get_mutators(mutators_to_run) From 89e2318fd8a3b8c2df9ab06a2b5a567faa21b62e Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 15:06:14 -0500 Subject: [PATCH 22/42] run tests before starting, abort if they don't pass --- slither/tools/mutator/__main__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 6147138ec..257ba5549 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -9,6 +9,7 @@ from slither import Slither from slither.tools.mutator.mutators import all_mutators from slither.utils.colors import blue, magenta +from slither.tools.mutator.utils.testing_generated_mutant import run_test_cmd from .mutators.abstract_mutator import AbstractMutator from .utils.command_line import output_mutators from .utils.file_handling import ( @@ -200,6 +201,12 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t CR_RR_list.insert(1, M) mutators_list = CR_RR_list + mutators_list + if not run_test_cmd(test_command, "", timeout): + logger.error(red("Test suite fails before mutation, aborting")) + return + else: + logger.info(green("Test suite passes, commencing mutation campaign")) + for filename in sol_file_list: # pylint: disable=too-many-nested-blocks file_name = os.path.split(filename)[1].split(".sol")[0] # slither object From 61242af57a87119b9efab21c46a45bc33f5c9d48 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 15:29:47 -0500 Subject: [PATCH 23/42] set smart default timeout --- slither/tools/mutator/__main__.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 257ba5549..88df24b75 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -1,14 +1,15 @@ import argparse import inspect import logging -import sys import os import shutil +import sys +import time from typing import Type, List, Any, Optional from crytic_compile import cryticparser from slither import Slither from slither.tools.mutator.mutators import all_mutators -from slither.utils.colors import blue, magenta +from slither.utils.colors import blue, green, magenta, red from slither.tools.mutator.utils.testing_generated_mutant import run_test_cmd from .mutators.abstract_mutator import AbstractMutator from .utils.command_line import output_mutators @@ -201,11 +202,24 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t CR_RR_list.insert(1, M) mutators_list = CR_RR_list + mutators_list - if not run_test_cmd(test_command, "", timeout): + # run and time tests, abort if they're broken + start_time = time.time() + if not run_test_cmd(test_command, "", 600): # use a very long timeout this first time logger.error(red("Test suite fails before mutation, aborting")) return + + elapsed_time = round(time.time() - start_time) + + # set default timeout + # default to twice as long as it usually takes to run the test suite + if timeout is None: + timeout = int(elapsed_time * 2) else: - logger.info(green("Test suite passes, commencing mutation campaign")) + timeout = int(timeout) + if timeout < elapsed_time: + logger.info(red(f"Provided timeout {timeout} is too short for tests that run in {elapsed_time} seconds")) + + logger.info(green(f"Test suite passes in {elapsed_time} seconds, commencing mutation campaign with a timeout of {timeout} seconds\n")) for filename in sol_file_list: # pylint: disable=too-many-nested-blocks file_name = os.path.split(filename)[1].split(".sol")[0] From b362a3c05788494765321f226001f289552175c3 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 15:52:09 -0500 Subject: [PATCH 24/42] force recompilation during initial timing test run --- slither/tools/mutator/__main__.py | 2 +- slither/tools/mutator/utils/testing_generated_mutant.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 88df24b75..dde8c606b 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -204,7 +204,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t # run and time tests, abort if they're broken start_time = time.time() - if not run_test_cmd(test_command, "", 600): # use a very long timeout this first time + if not run_test_cmd(test_command, "", 0): # no timeout during the first run logger.error(red("Test suite fails before mutation, aborting")) return diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 3cef50375..ff71a3cbb 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -34,13 +34,17 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None) -> bool elif "hardhat test" in cmd or "truffle test" in cmd and "--bail" not in cmd: cmd += " --bail" + if timeout == 0: + # add --forrce to ensure all contracts are recompiled w/out using cache + cmd += " --force" + try: result = subprocess.run( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=timeout, + timeout=timeout if timeout != 0 else None, check=False # True: Raises a CalledProcessError if the return code is non-zero ) From 34ab55f682e30274458ca93124197d26814de616 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 16:03:29 -0500 Subject: [PATCH 25/42] replace quick flag with comprehensive flag --- slither/tools/mutator/__main__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index dde8c606b..08c25f48f 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -99,8 +99,8 @@ def parse_args() -> argparse.Namespace: # flag to run full mutation based revert mutator output parser.add_argument( - "--quick", - help="to stop full mutation if revert mutator passes", + "--comprehensive", + help="continue testing minor mutations if severe mutants are uncaught", action="store_true", default=False, ) @@ -161,7 +161,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t very_verbose: Optional[bool] = args.very_verbose mutators_to_run: Optional[List[str]] = args.mutators_to_run contract_names: Optional[List[str]] = args.contract_names - quick_flag: Optional[bool] = args.quick + comprehensive_flag: Optional[bool] = args.comprehensive logger.info(blue(f"Starting Mutation Campaign in '{args.codebase}")) @@ -294,7 +294,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t logger.info(magenta(f"Running total: found {uncaught_mutant_counts[2]} uncaught tweak mutants (out of {total_mutant_counts[2]} that compile)")) dont_mutate_lines = lines_list - if not quick_flag: + if comprehensive_flag: dont_mutate_lines = [] except Exception as e: # pylint: disable=broad-except From e47f4e1b941279baff19151da1320fab40ba08c3 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 16:41:43 -0500 Subject: [PATCH 26/42] rm some obsolete code & polish logs --- slither/tools/mutator/__main__.py | 17 ++++++++--------- .../tools/mutator/mutators/abstract_mutator.py | 11 +++++------ .../mutator/utils/testing_generated_mutant.py | 8 ++++---- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 08c25f48f..06a4299df 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -8,9 +8,9 @@ from typing import Type, List, Any, Optional from crytic_compile import cryticparser from slither import Slither +from slither.tools.mutator.utils.testing_generated_mutant import run_test_cmd from slither.tools.mutator.mutators import all_mutators from slither.utils.colors import blue, green, magenta, red -from slither.tools.mutator.utils.testing_generated_mutant import run_test_cmd from .mutators.abstract_mutator import AbstractMutator from .utils.command_line import output_mutators from .utils.file_handling import ( @@ -181,12 +181,6 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t if os.path.exists(output_folder): shutil.rmtree(output_folder) - # set default timeout - if timeout is None: - timeout = 30 - else: - timeout = int(timeout) - # setting RR mutator as first mutator mutators_list = _get_mutators(mutators_to_run) @@ -202,12 +196,13 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t CR_RR_list.insert(1, M) mutators_list = CR_RR_list + mutators_list + logger.info(blue("Timing tests..")) + # run and time tests, abort if they're broken start_time = time.time() - if not run_test_cmd(test_command, "", 0): # no timeout during the first run + if not run_test_cmd(test_command, None, None): # no timeout or target_file during the first run logger.error(red("Test suite fails before mutation, aborting")) return - elapsed_time = round(time.time() - start_time) # set default timeout @@ -218,6 +213,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t timeout = int(timeout) if timeout < elapsed_time: logger.info(red(f"Provided timeout {timeout} is too short for tests that run in {elapsed_time} seconds")) + return logger.info(green(f"Test suite passes in {elapsed_time} seconds, commencing mutation campaign with a timeout of {timeout} seconds\n")) @@ -245,11 +241,13 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t target_contract = "SLITHER_SKIP_MUTATIONS" elif contract.name.lower() == file_name.lower(): target_contract = contract + if target_contract == "": logger.info( f"Cannot find contracts in file {filename}, try specifying them with --contract-names" ) continue + if target_contract == "SLITHER_SKIP_MUTATIONS": logger.debug(f"Skipping mutations in {filename}") continue @@ -349,4 +347,5 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t logger.info(blue(f"Finished Mutation Campaign in '{args.codebase}' \n")) + # endregion diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index 80665d1dc..e0901cae3 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -4,7 +4,6 @@ from slither.core.compilation_unit import SlitherCompilationUnit from slither.formatters.utils.patches import apply_patch, create_diff from slither.tools.mutator.utils.testing_generated_mutant import test_patch -from slither.utils.colors import yellow from slither.core.declarations import Contract logger = logging.getLogger("Slither-Mutate") @@ -81,11 +80,11 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: if "patches" not in all_patches: logger.debug("No patches found by %s", self.NAME) return ([0, 0, 0], [0, 0, 0], self.dont_mutate_line) + for file in all_patches["patches"]: # Note: This should only loop over a single file original_txt = self.slither.source_code[file].encode("utf8") patches = all_patches["patches"][file] patches.sort(key=lambda x: x["start"]) - logger.info(yellow(f"Mutating {file} with {self.NAME} \n")) for patch in patches: # test the patch patchWasCaught = test_patch( @@ -110,22 +109,22 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: else: self.uncaught_mutant_counts[2] += 1 - patched_txt,_ = apply_patch(original_txt, patch, 0) + patched_txt, _ = apply_patch(original_txt, patch, 0) diff = create_diff(self.compilation_unit, original_txt, patched_txt, file) if not diff: logger.info(f"Impossible to generate patch; empty {patches}") - # add valid mutant patches to a output file + # add uncaught mutant patches to a output file with open( self.output_folder + "/patches_file.txt", "a", encoding="utf8" ) as patches_file: patches_file.write(diff + "\n") + # count the total number of mutants that we were able to compile if patchWasCaught != 2: if self.NAME == "RR": - self.total_mutant_counts[0] += 1 - elif self.NAME == 'CR': + elif self.NAME == "CR": self.total_mutant_counts[1] += 1 else: self.total_mutant_counts[2] += 1 diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index ff71a3cbb..263c0562c 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -34,8 +34,8 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None) -> bool elif "hardhat test" in cmd or "truffle test" in cmd and "--bail" not in cmd: cmd += " --bail" - if timeout == 0: - # add --forrce to ensure all contracts are recompiled w/out using cache + if timeout is None: + # if no timeout, ensure all contracts are recompiled w/out using any cache cmd += " --force" try: @@ -44,7 +44,7 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None) -> bool shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=timeout if timeout != 0 else None, + timeout=timeout, check=False # True: Raises a CalledProcessError if the return code is non-zero ) @@ -54,7 +54,7 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None) -> bool result = None # or set result to a default value except KeyboardInterrupt: - logger.info("Ctrl-C received") + logger.info(yellow("Ctrl-C received")) if target_file is not None: logger.info("Restoring original files") reset_file(target_file) From 924252e4207df2afe312eba0489502cb63a136ae Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 16:49:54 -0500 Subject: [PATCH 27/42] run black reformatter --- slither/tools/mutator/__main__.py | 38 +++++++++++++++---- .../mutator/mutators/abstract_mutator.py | 20 +++++----- slither/tools/mutator/utils/file_handling.py | 1 + .../mutator/utils/testing_generated_mutant.py | 9 +++-- 4 files changed, 48 insertions(+), 20 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 06a4299df..38d54bbd1 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -200,7 +200,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t # run and time tests, abort if they're broken start_time = time.time() - if not run_test_cmd(test_command, None, None): # no timeout or target_file during the first run + if not run_test_cmd(test_command, None, None): # no timeout or target_file during the first run logger.error(red("Test suite fails before mutation, aborting")) return elapsed_time = round(time.time() - start_time) @@ -212,10 +212,18 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t else: timeout = int(timeout) if timeout < elapsed_time: - logger.info(red(f"Provided timeout {timeout} is too short for tests that run in {elapsed_time} seconds")) + logger.info( + red( + f"Provided timeout {timeout} is too short for tests that run in {elapsed_time} seconds" + ) + ) return - logger.info(green(f"Test suite passes in {elapsed_time} seconds, commencing mutation campaign with a timeout of {timeout} seconds\n")) + logger.info( + green( + f"Test suite passes in {elapsed_time} seconds, commencing mutation campaign with a timeout of {timeout} seconds\n" + ) + ) for filename in sol_file_list: # pylint: disable=too-many-nested-blocks file_name = os.path.split(filename)[1].split(".sol")[0] @@ -278,18 +286,34 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t total_mutant_counts[0] += total_counts[0] uncaught_mutant_counts[0] += uncaught_counts[0] if verbose: - logger.info(magenta(f"Mutator {m.NAME} found {uncaught_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)")) + logger.info( + magenta( + f"Mutator {m.NAME} found {uncaught_counts[0]} uncaught revert mutants (out of {total_counts[0]} that compile)" + ) + ) elif m.NAME == "CR": total_mutant_counts[1] += total_counts[1] uncaught_mutant_counts[1] += uncaught_counts[1] if verbose: - logger.info(magenta(f"Mutator {m.NAME} found {uncaught_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)")) + logger.info( + magenta( + f"Mutator {m.NAME} found {uncaught_counts[1]} uncaught comment mutants (out of {total_counts[1]} that compile)" + ) + ) else: total_mutant_counts[2] += total_counts[2] uncaught_mutant_counts[2] += uncaught_counts[2] if verbose: - logger.info(magenta(f"Mutator {m.NAME} found {uncaught_counts[2]} uncaught tweak mutants (out of {total_counts[2]} that compile)")) - logger.info(magenta(f"Running total: found {uncaught_mutant_counts[2]} uncaught tweak mutants (out of {total_mutant_counts[2]} that compile)")) + logger.info( + magenta( + f"Mutator {m.NAME} found {uncaught_counts[2]} uncaught tweak mutants (out of {total_counts[2]} that compile)" + ) + ) + logger.info( + magenta( + f"Running total: found {uncaught_mutant_counts[2]} uncaught tweak mutants (out of {total_mutant_counts[2]} that compile)" + ) + ) dont_mutate_lines = lines_list if comprehensive_flag: diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index e0901cae3..8e204bf2b 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -81,7 +81,7 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: logger.debug("No patches found by %s", self.NAME) return ([0, 0, 0], [0, 0, 0], self.dont_mutate_line) - for file in all_patches["patches"]: # Note: This should only loop over a single file + for file in all_patches["patches"]: # Note: This should only loop over a single file original_txt = self.slither.source_code[file].encode("utf8") patches = all_patches["patches"][file] patches.sort(key=lambda x: x["start"]) @@ -131,14 +131,16 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: if self.very_verbose: if self.NAME == "RR": - logger.info(f"Found {self.uncaught_mutant_counts[0]} uncaught revert mutants so far (out of {self.total_mutant_counts[0]} that compile)") + logger.info( + f"Found {self.uncaught_mutant_counts[0]} uncaught revert mutants so far (out of {self.total_mutant_counts[0]} that compile)" + ) elif self.NAME == "CR": - logger.info(f"Found {self.uncaught_mutant_counts[1]} uncaught comment mutants so far (out of {self.total_mutant_counts[1]} that compile)") + logger.info( + f"Found {self.uncaught_mutant_counts[1]} uncaught comment mutants so far (out of {self.total_mutant_counts[1]} that compile)" + ) else: - logger.info(f"Found {self.uncaught_mutant_counts[2]} uncaught tweak mutants so far (out of {self.total_mutant_counts[2]} that compile)") + logger.info( + f"Found {self.uncaught_mutant_counts[2]} uncaught tweak mutants so far (out of {self.total_mutant_counts[2]} that compile)" + ) - return ( - self.total_mutant_counts, - self.uncaught_mutant_counts, - self.dont_mutate_line - ) + return (self.total_mutant_counts, self.uncaught_mutant_counts, self.dont_mutate_line) diff --git a/slither/tools/mutator/utils/file_handling.py b/slither/tools/mutator/utils/file_handling.py index 08eb8f51c..8c435302c 100644 --- a/slither/tools/mutator/utils/file_handling.py +++ b/slither/tools/mutator/utils/file_handling.py @@ -49,6 +49,7 @@ def transfer_and_delete(files_dict: Dict) -> None: global_counter = {} + def create_mutant_file(file: str, rule: str) -> None: """function to create new mutant file""" try: diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 263c0562c..bc1562681 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -45,7 +45,7 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None) -> bool stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout, - check=False # True: Raises a CalledProcessError if the return code is non-zero + check=False, # True: Raises a CalledProcessError if the return code is non-zero ) except subprocess.TimeoutExpired: @@ -68,6 +68,7 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None) -> bool return False + # return 0 if uncaught, 1 if caught, and 2 if compilation fails def test_patch( # pylint: disable=too-many-arguments file: str, @@ -99,7 +100,7 @@ def test_patch( # pylint: disable=too-many-arguments ) ) reset_file(file) - return 0 # uncaught + return 0 # uncaught else: if very_verbose: logger.info( @@ -109,7 +110,7 @@ def test_patch( # pylint: disable=too-many-arguments ) reset_file(file) - return 2 # compile failure + return 2 # compile failure if verbose: logger.info( @@ -119,4 +120,4 @@ def test_patch( # pylint: disable=too-many-arguments ) reset_file(file) - return 1 # caught + return 1 # caught From fac704270a2ecf8101cc5b85281936f766ff22f4 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 17:26:11 -0500 Subject: [PATCH 28/42] log output if tests fail before mutating --- slither/tools/mutator/__main__.py | 7 ++++--- .../tools/mutator/utils/testing_generated_mutant.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 38d54bbd1..f02b84f53 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -163,7 +163,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t contract_names: Optional[List[str]] = args.contract_names comprehensive_flag: Optional[bool] = args.comprehensive - logger.info(blue(f"Starting Mutation Campaign in '{args.codebase}")) + logger.info(blue(f"Starting mutation campaign in {args.codebase}")) if paths_to_ignore: paths_to_ignore_list = paths_to_ignore.strip("][").split(",") @@ -200,8 +200,9 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t # run and time tests, abort if they're broken start_time = time.time() - if not run_test_cmd(test_command, None, None): # no timeout or target_file during the first run - logger.error(red("Test suite fails before mutation, aborting")) + # no timeout or target_file during the first run, but be verbose on failure + if not run_test_cmd(test_command, None, None, True): + logger.error(red("Test suite fails with mutations, aborting")) return elapsed_time = round(time.time() - start_time) diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index bc1562681..09094df08 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -21,7 +21,7 @@ def compile_generated_mutant(file_path: str, mappings: str) -> bool: return False -def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None) -> bool: +def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None, verbose: bool) -> bool: """ function to run codebase tests returns: boolean whether the tests passed or not @@ -64,7 +64,13 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None) -> bool # if result is 0 then it is an uncaught mutant because tests didn't fail if result: code = result.returncode - return code == 0 + if code == 0: + return True + + # If tests fail in verbose-mode, print both stdout and stderr for easier debugging + if verbose: + logger.info(yellow(result.stdout.decode('utf-8'))) + logger.info(red(result.stderr.decode('utf-8'))) return False @@ -92,7 +98,7 @@ def test_patch( # pylint: disable=too-many-arguments with open(file, "w", encoding="utf-8") as filepath: filepath.write(replaced_content) if compile_generated_mutant(file, mappings): - if run_test_cmd(command, timeout, file): + if run_test_cmd(command, timeout, file, False): create_mutant_file(file, generator_name) logger.info( red( From df4154d3ed4d7ad3c06200a95b872c9acc39b6e2 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 17:41:16 -0500 Subject: [PATCH 29/42] log total elapsed time --- slither/tools/mutator/__main__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index f02b84f53..67602f99a 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -370,7 +370,18 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t uncaught_mutant_counts[1] = 0 uncaught_mutant_counts[2] = 0 - logger.info(blue(f"Finished Mutation Campaign in '{args.codebase}' \n")) + # Print the total time elapsed in a human-readable time format + elapsed_time = round(time.time() - start_time) + hours, remainder = divmod(elapsed_time, 3600) + minutes, seconds = divmod(remainder, 60) + if hours > 0: + elapsed_string = f"{hours} {'hour' if hours == 1 else 'hours'}" + elif minutes > 0: + elapsed_string = f"{minutes} {'minute' if minutes == 1 else 'minutes'}" + else: + elapsed_string = f"{seconds} {'second' if seconds == 1 else 'seconds'}" + + logger.info(blue(f"Finished mutation testing assessment of '{args.codebase}' in {elapsed_string}\n")) # endregion From bb68df081883dd2bc8a5e34dcb1782f6ceaf6749 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 17:52:03 -0500 Subject: [PATCH 30/42] black format --- slither/tools/mutator/__main__.py | 4 +++- slither/tools/mutator/utils/testing_generated_mutant.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 67602f99a..5e5efba00 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -381,7 +381,9 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t else: elapsed_string = f"{seconds} {'second' if seconds == 1 else 'seconds'}" - logger.info(blue(f"Finished mutation testing assessment of '{args.codebase}' in {elapsed_string}\n")) + logger.info( + blue(f"Finished mutation testing assessment of '{args.codebase}' in {elapsed_string}\n") + ) # endregion diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 09094df08..d790f0f9e 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -69,8 +69,8 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None, verbose # If tests fail in verbose-mode, print both stdout and stderr for easier debugging if verbose: - logger.info(yellow(result.stdout.decode('utf-8'))) - logger.info(red(result.stderr.decode('utf-8'))) + logger.info(yellow(result.stdout.decode("utf-8"))) + logger.info(red(result.stderr.decode("utf-8"))) return False From 14bdd7cd366f613a2afa35bbb95ea1b4603b199c Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 16 Feb 2024 18:06:41 -0500 Subject: [PATCH 31/42] ignore pylint warning re too-many-branches --- slither/tools/mutator/mutators/abstract_mutator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index 8e204bf2b..2d1e68107 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -74,6 +74,7 @@ def _mutate(self) -> Dict: """TODO Documentation""" return {} + # pylint: disable=too-many-branches def mutate(self) -> Tuple[List[int], List[int], List[int]]: # call _mutate function from different mutators (all_patches) = self._mutate() From 5d36814734fbc336975e2d91af0d7ab059e32fea Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 7 Mar 2024 17:20:59 +0100 Subject: [PATCH 32/42] Improve file handling in Mutator --- slither/tools/mutator/__main__.py | 9 +- slither/tools/mutator/utils/file_handling.py | 98 ++++++++----------- .../mutator/utils/testing_generated_mutant.py | 18 ++-- 3 files changed, 57 insertions(+), 68 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 5e5efba00..fa62d93e7 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -5,6 +5,7 @@ import shutil import sys import time +from pathlib import Path from typing import Type, List, Any, Optional from crytic_compile import cryticparser from slither import Slither @@ -172,13 +173,14 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t paths_to_ignore_list = [] # get all the contracts as a list from given codebase - sol_file_list: List[str] = get_sol_file_list(args.codebase, paths_to_ignore_list) + sol_file_list: List[str] = get_sol_file_list(Path(args.codebase), paths_to_ignore_list) # folder where backup files and uncaught mutants are saved if output_dir is None: output_dir = "/mutation_campaign" - output_folder = os.getcwd() + output_dir - if os.path.exists(output_folder): + + output_folder = Path(output_dir).resolve() + if output_folder.is_dir(): shutil.rmtree(output_folder) # setting RR mutator as first mutator @@ -322,6 +324,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t except Exception as e: # pylint: disable=broad-except logger.error(e) + transfer_and_delete(files_dict) except KeyboardInterrupt: # transfer and delete the backup files if interrupted diff --git a/slither/tools/mutator/utils/file_handling.py b/slither/tools/mutator/utils/file_handling.py index 8c435302c..d77eb6573 100644 --- a/slither/tools/mutator/utils/file_handling.py +++ b/slither/tools/mutator/utils/file_handling.py @@ -1,89 +1,77 @@ import os import traceback -from typing import Dict, List +from typing import Dict, List, Union import logging +from pathlib import Path +import hashlib logger = logging.getLogger("Slither-Mutate") -duplicated_files = {} +HashedPath = str +backuped_files: Dict[str, HashedPath] = {} -def backup_source_file(source_code: Dict, output_folder: str) -> Dict: +def backup_source_file(source_code: Dict, output_folder: Path) -> Dict[str, HashedPath]: """ function to backup the source file returns: dictionary of duplicated files """ - os.makedirs(output_folder, exist_ok=True) - + output_folder.mkdir(exist_ok=True, parents=True) for file_path, content in source_code.items(): - directory, filename = os.path.split(file_path) - new_filename = f"{output_folder}/backup_{filename}" - new_file_path = os.path.join(directory, new_filename) + path_hash = hashlib.md5(bytes(file_path, "utf8")).hexdigest() + (output_folder / path_hash).write_text(content, encoding="utf8") - with open(new_file_path, "w", encoding="utf8") as new_file: - new_file.write(content) - duplicated_files[file_path] = new_file_path + backuped_files[file_path] = (output_folder / path_hash).as_posix() - return duplicated_files + return backuped_files -def transfer_and_delete(files_dict: Dict) -> None: +def transfer_and_delete(files_dict: Dict[str, HashedPath]) -> None: """function to transfer the original content to the sol file after campaign""" try: files_dict_copy = files_dict.copy() - for item, value in files_dict_copy.items(): - with open(value, "r", encoding="utf8") as duplicated_file: + for original_path, hashed_path in files_dict_copy.items(): + with open(hashed_path, "r", encoding="utf8") as duplicated_file: content = duplicated_file.read() - with open(item, "w", encoding="utf8") as original_file: + with open(original_path, "w", encoding="utf8") as original_file: original_file.write(content) - os.remove(value) + os.remove(hashed_path) # delete elements from the global dict - del duplicated_files[item] + del backuped_files[original_path] - except Exception as e: # pylint: disable=broad-except - logger.error(f"Error transferring content: {e}") + except FileNotFoundError as e: # pylint: disable=broad-except + logger.error(f"Error transferring content: %s", e) global_counter = {} -def create_mutant_file(file: str, rule: str) -> None: +def create_mutant_file(output_folder: Path, file: str, rule: str) -> None: """function to create new mutant file""" try: if rule not in global_counter: global_counter[rule] = 0 - _, filename = os.path.split(file) + + file_path = Path(file) # Read content from the duplicated file - with open(file, "r", encoding="utf8") as source_file: - content = source_file.read() + content = file_path.read_text(encoding="utf8") # Write content to the original file - mutant_name = filename.split(".")[0] - + mutant_name = file_path.stem # create folder for each contract - os.makedirs("mutation_campaign/" + mutant_name, exist_ok=True) - with open( - "mutation_campaign/" - + mutant_name - + "/" - + mutant_name - + "_" - + rule - + "_" - + str(global_counter[rule]) - + ".sol", - "w", - encoding="utf8", - ) as mutant_file: + mutation_dir = output_folder / mutant_name + mutation_dir.mkdir(parents=True, exist_ok=True) + + mutation_filename = f"{mutant_name}_{rule}_{global_counter[rule]}.sol" + with (mutation_dir / mutation_filename).open("w", encoding="utf8") as mutant_file: mutant_file.write(content) global_counter[rule] += 1 # reset the file - with open(duplicated_files[file], "r", encoding="utf8") as duplicated_file: - duplicate_content = duplicated_file.read() + duplicate_content = Path(backuped_files[file]).read_text("utf&") with open(file, "w", encoding="utf8") as source_file: source_file.write(duplicate_content) @@ -99,17 +87,17 @@ def reset_file(file: str) -> None: try: # directory, filename = os.path.split(file) # reset the file - with open(duplicated_files[file], "r", encoding="utf8") as duplicated_file: + with open(backuped_files[file], "r", encoding="utf8") as duplicated_file: duplicate_content = duplicated_file.read() with open(file, "w", encoding="utf8") as source_file: source_file.write(duplicate_content) except Exception as e: # pylint: disable=broad-except - logger.error(f"Error resetting file: {e}") + logger.error(f"Error resetting file: %s", e) -def get_sol_file_list(codebase: str, ignore_paths: List[str] | None) -> List[str]: +def get_sol_file_list(codebase: Path, ignore_paths: Union[List[str], None]) -> List[str]: """ function to get the contracts list returns: list of .sol files @@ -119,21 +107,13 @@ def get_sol_file_list(codebase: str, ignore_paths: List[str] | None) -> List[str ignore_paths = [] # if input is contract file - if os.path.isfile(codebase): - return [codebase] + if codebase.is_file(): + return [codebase.as_posix()] # if input is folder - if os.path.isdir(codebase): - directory = os.path.abspath(codebase) - for file in os.listdir(directory): - filename = os.path.join(directory, file) - if os.path.isfile(filename): - sol_file_list.append(filename) - elif os.path.isdir(filename): - _, dirname = os.path.split(filename) - if dirname in ignore_paths: - continue - for i in get_sol_file_list(filename, ignore_paths): - sol_file_list.append(i) + if codebase.is_dir(): + for file_name in codebase.rglob("*"): + if not any(part in ignore_paths for part in file_name.parts): + sol_file_list.append(file_name.as_posix()) return sol_file_list diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index d790f0f9e..685102f69 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -1,6 +1,8 @@ import logging import sys import subprocess +import tempfile +from pathlib import Path from typing import Dict import crytic_compile from slither.tools.mutator.utils.file_handling import create_mutant_file, reset_file @@ -97,15 +99,19 @@ def test_patch( # pylint: disable=too-many-arguments # Write the modified content back to the file with open(file, "w", encoding="utf-8") as filepath: filepath.write(replaced_content) + if compile_generated_mutant(file, mappings): if run_test_cmd(command, timeout, file, False): - create_mutant_file(file, generator_name) - logger.info( - red( - f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> UNCAUGHT" + + with tempfile.TemporaryDirectory() as tmpdirname: + create_mutant_file(Path(tmpdirname), file, generator_name) + logger.info( + red( + f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> UNCAUGHT" + ) ) - ) - reset_file(file) + reset_file(file) + return 0 # uncaught else: if very_verbose: From 4081125ec4f22bfe0d59f71cdf4db1a4bc231a65 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 8 Mar 2024 09:31:29 +0100 Subject: [PATCH 33/42] Fix wrong path --- slither/tools/mutator/__main__.py | 2 +- slither/tools/mutator/mutators/abstract_mutator.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index fa62d93e7..c89b5973b 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -177,7 +177,7 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t # folder where backup files and uncaught mutants are saved if output_dir is None: - output_dir = "/mutation_campaign" + output_dir = "./mutation_campaign" output_folder = Path(output_dir).resolve() if output_folder.is_dir(): diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index 2d1e68107..77cde2ffa 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -1,5 +1,6 @@ import abc import logging +from pathlib import Path from typing import Optional, Dict, Tuple, List from slither.core.compilation_unit import SlitherCompilationUnit from slither.formatters.utils.patches import apply_patch, create_diff @@ -29,7 +30,7 @@ def __init__( # pylint: disable=too-many-arguments solc_remappings: str | None, verbose: bool, very_verbose: bool, - output_folder: str, + output_folder: Path, dont_mutate_line: List[int], rate: int = 10, seed: Optional[int] = None, @@ -116,9 +117,7 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: logger.info(f"Impossible to generate patch; empty {patches}") # add uncaught mutant patches to a output file - with open( - self.output_folder + "/patches_file.txt", "a", encoding="utf8" - ) as patches_file: + with (self.output_folder / "patches_files.txt").open("a", encoding="utf8") as patches_file: patches_file.write(diff + "\n") # count the total number of mutants that we were able to compile From f46574e7268cb1701c8d760834483e6cd6894e22 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 8 Mar 2024 10:21:26 +0100 Subject: [PATCH 34/42] Run formatters --- slither/tools/mutator/mutators/abstract_mutator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index 77cde2ffa..da6983583 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -117,7 +117,9 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: logger.info(f"Impossible to generate patch; empty {patches}") # add uncaught mutant patches to a output file - with (self.output_folder / "patches_files.txt").open("a", encoding="utf8") as patches_file: + with (self.output_folder / "patches_files.txt").open( + "a", encoding="utf8" + ) as patches_file: patches_file.write(diff + "\n") # count the total number of mutants that we were able to compile From 354f5ba4573dca336af758e40390aa3d5243e0f8 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 8 Mar 2024 10:23:51 +0100 Subject: [PATCH 35/42] Remove os module usage --- slither/tools/mutator/utils/file_handling.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/slither/tools/mutator/utils/file_handling.py b/slither/tools/mutator/utils/file_handling.py index d77eb6573..c74481511 100644 --- a/slither/tools/mutator/utils/file_handling.py +++ b/slither/tools/mutator/utils/file_handling.py @@ -1,4 +1,3 @@ -import os import traceback from typing import Dict, List, Union import logging @@ -37,7 +36,7 @@ def transfer_and_delete(files_dict: Dict[str, HashedPath]) -> None: with open(original_path, "w", encoding="utf8") as original_file: original_file.write(content) - os.remove(hashed_path) + Path(hashed_path).unlink() # delete elements from the global dict del backuped_files[original_path] @@ -85,7 +84,6 @@ def create_mutant_file(output_folder: Path, file: str, rule: str) -> None: def reset_file(file: str) -> None: """function to reset the file""" try: - # directory, filename = os.path.split(file) # reset the file with open(backuped_files[file], "r", encoding="utf8") as duplicated_file: duplicate_content = duplicated_file.read() From c37c50647cdf0b3ffe2a3ffc8efb039c331a7224 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 8 Mar 2024 10:27:46 +0100 Subject: [PATCH 36/42] Fix typo in encoding --- slither/tools/mutator/utils/file_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slither/tools/mutator/utils/file_handling.py b/slither/tools/mutator/utils/file_handling.py index c74481511..31cd697dc 100644 --- a/slither/tools/mutator/utils/file_handling.py +++ b/slither/tools/mutator/utils/file_handling.py @@ -70,7 +70,7 @@ def create_mutant_file(output_folder: Path, file: str, rule: str) -> None: global_counter[rule] += 1 # reset the file - duplicate_content = Path(backuped_files[file]).read_text("utf&") + duplicate_content = Path(backuped_files[file]).read_text("utf8") with open(file, "w", encoding="utf8") as source_file: source_file.write(duplicate_content) From 4d2ab83d44a717ac5e7eecea2518491b2826ec47 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 8 Mar 2024 10:32:45 +0100 Subject: [PATCH 37/42] Save mutant file in mutation_campaign directory --- .../mutator/mutators/abstract_mutator.py | 1 + .../mutator/utils/testing_generated_mutant.py | 19 +++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index da6983583..69c77a4ca 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -90,6 +90,7 @@ def mutate(self) -> Tuple[List[int], List[int], List[int]]: for patch in patches: # test the patch patchWasCaught = test_patch( + self.output_folder, file, patch, self.test_command, diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index 685102f69..4aef72a6a 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -1,9 +1,8 @@ import logging import sys import subprocess -import tempfile from pathlib import Path -from typing import Dict +from typing import Dict, Union import crytic_compile from slither.tools.mutator.utils.file_handling import create_mutant_file, reset_file from slither.utils.colors import green, red, yellow @@ -79,12 +78,13 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None, verbose # return 0 if uncaught, 1 if caught, and 2 if compilation fails def test_patch( # pylint: disable=too-many-arguments + output_folder: Path, file: str, patch: Dict, command: str, generator_name: str, timeout: int, - mappings: str | None, + mappings: Union[str, None], verbose: bool, very_verbose: bool, ) -> int: @@ -103,14 +103,13 @@ def test_patch( # pylint: disable=too-many-arguments if compile_generated_mutant(file, mappings): if run_test_cmd(command, timeout, file, False): - with tempfile.TemporaryDirectory() as tmpdirname: - create_mutant_file(Path(tmpdirname), file, generator_name) - logger.info( - red( - f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> UNCAUGHT" - ) + create_mutant_file(output_folder, file, generator_name) + logger.info( + red( + f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> UNCAUGHT" ) - reset_file(file) + ) + reset_file(file) return 0 # uncaught else: From 4664c97abced3f6fdf8e8b445c44f780268cb1e4 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 8 Mar 2024 11:19:31 +0100 Subject: [PATCH 38/42] Fix a bug where contract-names parameter where not properly handled. --- slither/tools/mutator/__main__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index c89b5973b..19af71f8c 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -161,7 +161,6 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t verbose: Optional[bool] = args.verbose very_verbose: Optional[bool] = args.very_verbose mutators_to_run: Optional[List[str]] = args.mutators_to_run - contract_names: Optional[List[str]] = args.contract_names comprehensive_flag: Optional[bool] = args.comprehensive logger.info(blue(f"Starting mutation campaign in {args.codebase}")) @@ -172,6 +171,10 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t else: paths_to_ignore_list = [] + contract_names: List[str] = [] + if args.contract_names: + contract_names = args.contract_names.split(",") + # get all the contracts as a list from given codebase sol_file_list: List[str] = get_sol_file_list(Path(args.codebase), paths_to_ignore_list) @@ -242,16 +245,16 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t dont_mutate_lines = [] # mutation - target_contract = "" try: for compilation_unit_of_main_file in sl.compilation_units: + target_contract = "SLITHER_SKIP_MUTATIONS" if contract_names else "" for contract in compilation_unit_of_main_file.contracts: - if contract_names is not None and contract.name in contract_names: + if contract.name in contract_names: target_contract = contract - elif contract_names is not None and contract.name not in contract_names: - target_contract = "SLITHER_SKIP_MUTATIONS" - elif contract.name.lower() == file_name.lower(): + break + elif not contract_names and contract.name.lower() == file_name.lower(): target_contract = contract + break if target_contract == "": logger.info( From 59327fa97e5c27b0b7edb9ecb72064325a0a09bd Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 8 Mar 2024 11:19:52 +0100 Subject: [PATCH 39/42] Fix a bug where also directories were iterated on --- slither/tools/mutator/utils/file_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slither/tools/mutator/utils/file_handling.py b/slither/tools/mutator/utils/file_handling.py index 31cd697dc..15f2b9506 100644 --- a/slither/tools/mutator/utils/file_handling.py +++ b/slither/tools/mutator/utils/file_handling.py @@ -110,7 +110,7 @@ def get_sol_file_list(codebase: Path, ignore_paths: Union[List[str], None]) -> L # if input is folder if codebase.is_dir(): - for file_name in codebase.rglob("*"): + for file_name in codebase.rglob("*.sol"): if not any(part in ignore_paths for part in file_name.parts): sol_file_list.append(file_name.as_posix()) From c2ff06ee2d9599cebe04e90db51f9f0f63979665 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 8 Mar 2024 14:02:33 +0100 Subject: [PATCH 40/42] Only mutate contracts once. --- slither/tools/mutator/__main__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 19af71f8c..15a695a9e 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -148,7 +148,7 @@ def __call__( ################################################################################### -def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,too-many-locals +def main() -> None: # pylint: disable=too-many-statements,too-many-branches,too-many-locals args = parse_args() # arguments @@ -231,6 +231,9 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t ) ) + # Keep a list of all already mutated contracts so we don't mutate them twice + mutated_contracts: List[str] = [] + for filename in sol_file_list: # pylint: disable=too-many-nested-blocks file_name = os.path.split(filename)[1].split(".sol")[0] # slither object @@ -245,11 +248,11 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t dont_mutate_lines = [] # mutation + target_contract = "SLITHER_SKIP_MUTATIONS" if contract_names else "" try: for compilation_unit_of_main_file in sl.compilation_units: - target_contract = "SLITHER_SKIP_MUTATIONS" if contract_names else "" for contract in compilation_unit_of_main_file.contracts: - if contract.name in contract_names: + if contract.name in contract_names and contract.name not in mutated_contracts: target_contract = contract break elif not contract_names and contract.name.lower() == file_name.lower(): @@ -272,6 +275,8 @@ def main() -> (None): # pylint: disable=too-many-statements,too-many-branches,t logger.debug(f"Skipping mutations on interface {filename}") continue + # Add our target to the mutation list + mutated_contracts.append(target_contract.name) logger.info(blue(f"Mutating contract {target_contract}")) for M in mutators_list: m = M( From 84efb2f11e303779f7f449982c0bd5edabce54b7 Mon Sep 17 00:00:00 2001 From: bohendo Date: Thu, 29 Feb 2024 15:07:23 -0500 Subject: [PATCH 41/42] omit --force flag in hardhat tests --- slither/tools/mutator/utils/testing_generated_mutant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index d790f0f9e..22ee20e06 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -34,7 +34,7 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None, verbose elif "hardhat test" in cmd or "truffle test" in cmd and "--bail" not in cmd: cmd += " --bail" - if timeout is None: + if timeout is None and "hardhat" not in cmd: # hardhat doesn't support --force flag on tests # if no timeout, ensure all contracts are recompiled w/out using any cache cmd += " --force" From 9dee8a2a50a539cdf15867016f50c19bcadec02c Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 29 Mar 2024 15:08:29 -0400 Subject: [PATCH 42/42] fix lint/formatting problems --- slither/tools/mutator/__main__.py | 2 +- slither/tools/mutator/utils/file_handling.py | 4 ++-- slither/tools/mutator/utils/testing_generated_mutant.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index 15a695a9e..8a7ce3e1a 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -255,7 +255,7 @@ def main() -> None: # pylint: disable=too-many-statements,too-many-branches,too if contract.name in contract_names and contract.name not in mutated_contracts: target_contract = contract break - elif not contract_names and contract.name.lower() == file_name.lower(): + if not contract_names and contract.name.lower() == file_name.lower(): target_contract = contract break diff --git a/slither/tools/mutator/utils/file_handling.py b/slither/tools/mutator/utils/file_handling.py index 15f2b9506..7c02ce099 100644 --- a/slither/tools/mutator/utils/file_handling.py +++ b/slither/tools/mutator/utils/file_handling.py @@ -42,7 +42,7 @@ def transfer_and_delete(files_dict: Dict[str, HashedPath]) -> None: del backuped_files[original_path] except FileNotFoundError as e: # pylint: disable=broad-except - logger.error(f"Error transferring content: %s", e) + logger.error("Error transferring content: %s", e) global_counter = {} @@ -92,7 +92,7 @@ def reset_file(file: str) -> None: source_file.write(duplicate_content) except Exception as e: # pylint: disable=broad-except - logger.error(f"Error resetting file: %s", e) + logger.error("Error resetting file: %s", e) def get_sol_file_list(codebase: Path, ignore_paths: Union[List[str], None]) -> List[str]: diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index a1f54df3b..39e7d39de 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -35,7 +35,7 @@ def run_test_cmd(cmd: str, timeout: int | None, target_file: str | None, verbose elif "hardhat test" in cmd or "truffle test" in cmd and "--bail" not in cmd: cmd += " --bail" - if timeout is None and "hardhat" not in cmd: # hardhat doesn't support --force flag on tests + if timeout is None and "hardhat" not in cmd: # hardhat doesn't support --force flag on tests # if no timeout, ensure all contracts are recompiled w/out using any cache cmd += " --force"