Merge pull request #25 from tahiat/build

running the gradle script to produce the issue stack

Keyvalue.py

@@ -17,3 +17,6 @@ class JsonKeys(Enum):
     NOTE = 'note'
     INNER_CLASS = 'inner_class'
     NON_PRIMARY_CLASS = 'non_primary_class'
+    BUG_TYPE = 'bug_type'
+    CHECKER_QUAL_REQURIED = 'checker_qual_required'
+    BUG_PATTERN = 'bug_pattern'

Result.py

@@ -9,4 +9,11 @@ def __init__(self, name, status, reason):
         '''
         self.name = name
         self.status = status
-        self.reason = reason
+        self.reason = reason
+        self.preservation_status = False
+
+    def set_preservation_status(self, status):
+        '''
+        status True if the minimized program preserve the target behavior
+        '''
+        self.preservation_status = status

exception_data.py

@@ -0,0 +1,12 @@
+class ExceptionData:
+    def __init__(self, exception_class: str, exception: str, stack_trace: list):
+        '''
+        Constructor of the class
+        Parameters:
+            exception_class: class of the exception
+            exception: which exception was raised
+            strack_trace: first 5 lines of the stack trace
+        '''
+        self.exception_class = exception_class
+        self.exception = exception
+        self.stack_trace = stack_trace

main.py

@@ -1,11 +1,13 @@
 import json
+import re
 import os
 import sys
 import subprocess
 import shutil
 from Keyvalue import JsonKeys
 from Result import Result
 from report_builder import TableGenerator
+from exception_data import ExceptionData
 
 issue_folder_dir = 'ISSUES'
 specimin_input = 'input'
@@ -15,6 +17,7 @@
 TIMEOUT_DURATION = 300
 specimin_env_var = "SPECIMIN"
 json_status_file_name = "target_status.json"
+minimized_program_build_log_file = "build_log.txt"
 
 def read_json_from_file(file_path):
     '''
@@ -81,12 +84,8 @@ def create_issue_directory(issue_container_dir, issue_id):
     os.makedirs(issue_directory_name, exist_ok=True)
 
     specimin_input_dir = os.path.join(issue_directory_name, specimin_input)
-    specimin_output_dir = os.path.join(issue_directory_name, specimin_output)
-
     os.makedirs(specimin_input_dir, exist_ok=True)
-    if os.path.exists(specimin_output):
-        shutil.rmtree(specimin_output)
-    os.makedirs(specimin_output_dir, exist_ok=True)
+
     return specimin_input_dir
 
 
@@ -212,7 +211,8 @@ def clone_specimin(path_to_clone, url):
 def build_specimin_command(project_name: str,
                            target_base_dir_path: str,
                            root_dir: str,  
-                           targets: list):
+                           targets: list,
+                           jar_path: str = ""):
     '''
     Build the gradle command to execute Specimin on target project
 
@@ -237,7 +237,11 @@ def build_specimin_command(project_name: str,
     if not os.path.isabs(target_base_dir_path):
         raise ValueError("Invalid argument: target_base_dir_path must be an absolute path")
 
-    output_dir = os.path.join(target_base_dir_path, specimin_output)
+    output_dir = os.path.join(target_base_dir_path, specimin_output, project_name, "src", "main", "java")
+
+    if os.path.exists(output_dir):
+        shutil.rmtree(output_dir)
+
     root_dir = os.path.join(target_base_dir_path, specimin_input, project_name, root_dir)
     root_dir = root_dir.rstrip('/') + os.sep
 
@@ -271,16 +275,19 @@ def build_specimin_command(project_name: str,
 
     output_dir_subcommand = "--outputDirectory" + " " + f"\"{output_dir}\""
     root_dir_subcommand = "--root" + " " + f"\"{root_dir}\""
-
     target_file_subcommand = ""
     for file in target_file_list:
         target_file_subcommand += "--targetFile" + " " + f"\"{file}\""
 
     target_method_subcommand = ""
     for method in target_method_list:
         target_method_subcommand += "--targetMethod" + " " + f"\"{method}\""
+
+    jar_path_subcommand = ""
+    if jar_path:
+        jar_path_subcommand = " --jarPath" + " " + f"\"{jar_path}\""
 
-    command_args = root_dir_subcommand + " " + output_dir_subcommand + " " + target_file_subcommand + " " + target_method_subcommand
+    command_args = root_dir_subcommand + " " + output_dir_subcommand + " " + target_file_subcommand + " " + target_method_subcommand + jar_path_subcommand
     command = "./gradlew" + " " + "run" + " " + "--args=" + f"\'{command_args}\'"
 
     return command
@@ -336,27 +343,207 @@ def performEvaluation(issue_data) -> Result:
     url = issue_data[JsonKeys.URL.value]
     branch = issue_data[JsonKeys.BRANCH.value]
     commit_hash = issue_data[JsonKeys.COMMIT_HASH.value]
+    qual_jar_required = issue_data[JsonKeys.CHECKER_QUAL_REQURIED.value]
+    qual_jar_dir = ""
 
     issue_folder_abs_dir = os.path.abspath(issue_folder_dir)
     input_dir = create_issue_directory(issue_folder_abs_dir, issue_id)
 
     get_target_data(url, branch, commit_hash, input_dir) 
 
     repo_name = get_repository_name(url)
+    if qual_jar_required:
+        qual_jar_dir = os.path.join(issue_folder_abs_dir, issue_id, specimin_input, repo_name, specimin_project_name)
     specimin_command = ""
     result: Result = None
     specimin_path = get_specimin_env_var()
+    specimin_command = build_specimin_command(repo_name, os.path.join(issue_folder_abs_dir, issue_id), issue_data[JsonKeys.ROOT_DIR.value], issue_data[JsonKeys.TARGETS.value], qual_jar_dir if os.path.exists(qual_jar_dir) else "")
+
     if specimin_path is not None and os.path.exists(specimin_path):
-        specimin_command = build_specimin_command(repo_name, os.path.join(issue_folder_abs_dir, issue_id), issue_data[JsonKeys.ROOT_DIR.value], issue_data[JsonKeys.TARGETS.value])
         result = run_specimin(issue_id ,specimin_command, specimin_path)
     else:
-        specimin_command = build_specimin_command(repo_name, os.path.join(issue_folder_abs_dir, issue_id),issue_data[JsonKeys.ROOT_DIR.value], issue_data[JsonKeys.TARGETS.value])
         result = run_specimin(issue_id ,specimin_command, os.path.join(issue_folder_abs_dir, specimin_project_name))
 
     print(f"{result.name} - {result.status}")
+
+    # build script is shipped with input program. It exists in the "specimin" directory of the input program's root directory.
+    # Coping the build script to the output directory of the minimized program.
+    build_script_path = os.path.join(issue_folder_abs_dir, issue_id, specimin_input, repo_name, specimin_project_name, "build.gradle")
+
+    if not os.path.exists(build_script_path): #TODO: when finish adding build script, raise exception to indicate missing build script
+        return result
+    build_script_destination_path = os.path.join(issue_folder_abs_dir, issue_id, specimin_output, repo_name, "build.gradle")
+
+    copy_build_script = f"cp {build_script_path} {build_script_destination_path}"
+    subprocess.run(copy_build_script, shell=True)
+
+    #../ISSUES/cf-xx/output/projectname/build_log.txt
+    log_file = os.path.join(issue_folder_abs_dir, issue_id, specimin_output, repo_name, minimized_program_build_log_file)
+
+    if os.path.exists(log_file):
+        os.remove(log_file)
+
+    # Open the log file in write mode
+    min_prgrm_build_status = None
+    with open(log_file, "w") as log_file_obj:
+        min_prgrm_build_status = subprocess.run(f"./gradlew -b  {build_script_destination_path} compileJava", cwd = os.path.abspath("resources"), shell=True, stderr=log_file_obj)
+        print(f"{issue_id} Minimized program gradle build status = {min_prgrm_build_status}")
+
+    expected_log_file = os.path.join(issue_folder_abs_dir, issue_id, specimin_input, repo_name, specimin_project_name, "expected_log.txt")
+    if (JsonKeys.BUG_TYPE.value in issue_data and issue_data[JsonKeys.BUG_TYPE.value] == "crash" and min_prgrm_build_status.returncode != 0):
+        status = compare_crash_log(expected_log_file, log_file)
+        result.set_preservation_status(status)
+    elif (JsonKeys.BUG_TYPE.value in issue_data and issue_data[JsonKeys.BUG_TYPE.value] == "error"):
+        status = compare_error_log(expected_log_file, log_file, issue_data[JsonKeys.BUG_PATTERN.value])
+        result.set_preservation_status(status)
+    elif (JsonKeys.BUG_TYPE.value in issue_data and issue_data[JsonKeys.BUG_TYPE.value] == "false_positive"):
+        status = compare_false_positive_log(expected_log_file, log_file, issue_data[JsonKeys.BUG_PATTERN.value])
+        result.set_preservation_status(status)
+    elif (JsonKeys.BUG_TYPE.value in issue_data and issue_data[JsonKeys.BUG_TYPE.value] == "semi_crash"):
+        status = compare_semi_crash(expected_log_file, log_file, issue_data[JsonKeys.BUG_PATTERN.value])
+        result.set_preservation_status(status)
     return result
 
 
+def compare_semi_crash(expected_log_path, actual_log_path, bug_pattern_data):
+    with open(expected_log_path, "r") as file:
+        expected_content = file.read()
+
+    with open(actual_log_path, "r") as file:
+        actual_content = file.read()
+
+    logs_to_match = []
+
+    for key in bug_pattern_data:
+        pattern = bug_pattern_data[key]
+        content = re.search(pattern, expected_content).group(1)
+        logs_to_match.append(content)
+    return all(string in actual_content for string in logs_to_match)
+
+
+def compare_false_positive_log(expected_log_path, actual_log_path,  bug_pattern_data):
+    with open(expected_log_path, "r") as file:
+        expected_content = file.read()
+
+    with open(actual_log_path, "r") as file:
+        actual_content = file.read()
+
+    file_pattern = bug_pattern_data["file_pattern"]
+    error_pattern = bug_pattern_data["error_pattern"]
+    source_pattern = bug_pattern_data["source_pattern"]
+    found_pattern = bug_pattern_data["found_pattern"]
+    required_pattern = bug_pattern_data["required_pattern"]
+
+    java_file = re.search(file_pattern, expected_content).group(1)
+    error_message = re.search(error_pattern, expected_content).group(1)
+    code_triggered_bug = re.search(source_pattern, expected_content).group(1)
+    found_type = re.search(found_pattern, expected_content).group(1)
+    required_type = re.search(required_pattern, expected_content).group(1)
+
+    return java_file in actual_content and error_message in actual_content and code_triggered_bug in actual_content and found_type in actual_content and required_type in actual_content
+
+
+def compare_error_log(expected_log_path, actual_log_path, bug_pattern_data):
+    '''
+    Compare the error log of the minimized program with the expected error log
+    '''
+    with open(expected_log_path, "r") as file:
+        expected_content = file.read()
+
+    with open(actual_log_path, "r") as file:
+        actual_content = file.read()
+
+    file_pattern = bug_pattern_data["file_pattern"]
+    error_pattern = bug_pattern_data["error_pattern"]
+    source_pattern = bug_pattern_data["source_pattern"]
+    reason_pattern = bug_pattern_data["reason_pattern"]
+
+    error_file = re.search(file_pattern, expected_content).group(1)
+    error_message = re.search(error_pattern, expected_content).group(1)
+    error_source = re.search(source_pattern, expected_content).group(1)
+    error_reason = re.search(reason_pattern, expected_content).group(1)
+
+    return error_file in actual_content and error_message in actual_content and error_source in actual_content and error_reason in actual_content
+
+
+def get_exception_data(log_file_data_list: list):
+    '''
+    Parse the exception data from the log file
+
+    Returns:
+        exception_data (ExceptionData): exception data
+    '''
+
+    return_data = []
+    cf_crash_line = [line_no for line_no, line in enumerate(log_file_data_list) if line.lstrip().startswith('; The Checker Framework crashed.')]
+    if len(cf_crash_line) == 0:
+        return None
+
+    for line_no in cf_crash_line: # if multiple crash location found, one shoud match exactly with the expected crash information
+        crashed_class_name_line = -1
+        for i in range(line_no, line_no + 5): # should be immediate next line of crash line
+            if log_file_data_list[i].strip().startswith("Compilation unit:"):
+                crashed_class_name_line = i
+                break
+        if crashed_class_name_line == -1:
+            continue  # start looking for next crash location
+        class_name_abs_path = log_file_data_list[crashed_class_name_line].split(" ")[-1]
+        crashed_class_name = os.path.basename(class_name_abs_path)
+
+        exception_line = -1
+        for i in range(crashed_class_name_line, crashed_class_name_line + 5): # should be immediate next line of crash line
+            if log_file_data_list[i].strip().startswith("Exception:"):
+                exception_line = i
+                break
+        exception_stack = [] #compare it with actual stack trace
+        exception_line_str = log_file_data_list[exception_line] #Exception: java.lang.NullPointerException; java.lang.NullPointerException
+        exception_line_sub_str = (exception_line_str[exception_line_str.index("Exception:") + 10:]).split()[0] # java.lang.NullPointerException; java.lang.NullPointerException
+        exception_cause = re.sub(r'^[^a-zA-Z]+|[^a-zA-Z]+$', '', exception_line_sub_str) # java.lang.NullPointerException
+        for i in range(exception_line + 1, exception_line + 6):
+            if log_file_data_list[i].lstrip().startswith("at"):
+                exception_stack.append(log_file_data_list[i].split()[-1].strip())
+
+        if crashed_class_name != None and exception_cause != None and len(exception_stack) > 0:
+            exception_data = ExceptionData(crashed_class_name, exception_cause, exception_stack)
+            return_data.append(exception_data)
+
+    return return_data
+
+def compare_crash_log(expected_log_path, actual_log_path):
+    '''
+    Compare the crash log of the minimized program with the expected crash log
+    '''
+
+    with open(expected_log_path, "r") as file:
+        expected_content = file.read()
+
+    with open(actual_log_path, "r") as file:
+        actual_content = file.read()
+
+    expected_lines = expected_content.split('\n')
+    actual_lines = actual_content.split('\n')
+
+    expected_crash_datas = get_exception_data(expected_lines) # there should be 1 crash data
+    actual_crash_data = get_exception_data(actual_lines)
+
+    if expected_crash_datas != None or len(expected_crash_datas) != 0:
+        expected_crash_data = expected_crash_datas[0]
+    else:
+        return False # no crash data found in the expected log file
+
+    is_crash_matched = True
+    for data in actual_crash_data:
+        is_crash_matched = True
+        if expected_crash_data.exception != data.exception or expected_crash_data.exception_class != data.exception_class:
+            is_crash_matched = False
+            continue
+        if expected_crash_data.stack_trace != data.stack_trace:
+            is_crash_matched = False
+            continue
+
+    return is_crash_matched
+
+
 def main():
     '''
     Main method of the script. It iterates over the json data and perform minimization for each cases.   
@@ -383,7 +570,7 @@ def main():
 
     parsed_data = read_json_from_file(json_file_path)
 
-    evaluation_results: list[Result] = []
+    evaluation_results = []
     json_status: dict[str, str] = {} # Contains PASS/FAIL status of targets to be printed as a json file 
     if parsed_data:
         for issue in parsed_data:
@@ -392,8 +579,7 @@ def main():
             result = performEvaluation(issue)
             evaluation_results.append(result)
             json_status[issue_id] = result.status
-            print((f"{issue_id} <========= execution Ends."))
-
+            print((f"{issue_id} <========= execution Ends."))            
 
     report_generator: TableGenerator = TableGenerator(evaluation_results)
     report_generator.generateTable()
@@ -404,12 +590,15 @@ def main():
         json.dump(json_status, json_file, indent= 2)
 
     print("\n\n\n\n")
-    print(f"issue_name    |    status    |    reason")
+    print(f"issue_name    |    status    |  Fail reason  | preservation_status")
     print("--------------------------------------------")
     case = 1
     for minimization_result in evaluation_results:
-        print(f"({case}){minimization_result.name}    |    {minimization_result.status}     |    {minimization_result.reason}")
+        print(f"({case}){minimization_result.name}    |    {minimization_result.status}     |    {minimization_result.reason}      | {minimization_result.preservation_status}")
         case +=1
+
+
+
 
 
 if __name__ == "__main__":

resources/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists