Revise summary JSON's representation and reduce compiler-crash test cases

README.md

@@ -33,10 +33,22 @@ export LLVM_VERSION=17.0.4
 ```
 cd ${DREDD_EXPERIMENTS_ROOT}
 git clone --recursive https://github.com/mc-imperial/dredd.git
+pushd dredd/third_party/clang+llvm
+    OS=ubuntu-22.04
+    DREDD_LLVM_TAG=17.0.6
+    curl -Lo clang+llvm.tar.xz "https://github.com/llvm/llvm-project/releases/download/llvmorg-${DREDD_LLVM_TAG}/clang+llvm-${DREDD_LLVM_TAG}-x86_64-linux-gnu-${OS}.tar.xz"
+    tar xf clang+llvm.tar.xz
+    mv clang+llvm-${DREDD_LLVM_TAG}-x86_64-linux-gnu-${OS}/* .
+    rm clang+llvm.tar.xz
+popd
+
+# (Optional) For reproducibility, checkout the dredd version used below
 pushd dredd
-    OS=ubuntu-22.04 DREDD_LLVM_SUFFIX=-stock-clang .github/workflows/install_clang.sh
+git checkout 2074c34a701211777554e4d2d6acdbb8fc1166f2
 popd
-cmake -S dredd -B dredd/build -G Ninja -DCMAKE_C_COMPILER=clang-15 -DCMAKE_CXX_COMPILER=clang++-15
+
+DREDD_COMPILER_PATH=${DREDD_EXPERIMENTS_ROOT}/dredd/third_party/clang+llvm/bin
+cmake -S dredd -B dredd/build -G Ninja -DCMAKE_C_COMPILER=${DREDD_COMPILER_PATH}/clang -DCMAKE_CXX_COMPILER=${DREDD_COMPILER_PATH}/clang++
 cmake --build dredd/build --target dredd
 cp dredd/build/src/dredd/dredd dredd/third_party/clang+llvm/bin/
 ```
@@ -70,9 +82,9 @@ do
   SOURCE_DIR=llvm-${LLVM_VERSION}-${kind}/llvm
   BUILD_DIR=llvm-${LLVM_VERSION}-${kind}-build
   mkdir ${BUILD_DIR}
-  cmake -S "${SOURCE_DIR}" -B "${BUILD_DIR}" -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_FLAGS="-w" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang" -DCMAKE_C_COMPILER=clang-15 -DCMAKE_CXX_COMPILER=clang++-15
+  cmake -S "${SOURCE_DIR}" -B "${BUILD_DIR}" -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_FLAGS="-w" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang" -DCMAKE_C_COMPILER=${DREDD_COMPILER_PATH}/clang -DCMAKE_CXX_COMPILER=${DREDD_COMPILER_PATH}/clang++
   # Build something minimal to ensure all auto-generated pieces of code are created.
-  cmake --build "${BUILD_DIR}" --target LLVMCore
+  cmake --build "${BUILD_DIR}" --target all
 done
 ```
 
@@ -85,6 +97,9 @@ export DREDD_EXECUTABLE=${DREDD_EXPERIMENTS_ROOT}/dredd/third_party/clang+llvm/b
 Mutate all `.cpp` files under `InstCombine` in the copy of LLVM designated for mutation:
 
 ```
+# (Optional) `sort` depend on locale, for reproducibility:
+export LC_ALL=C
+
 cd ${DREDD_EXPERIMENTS_ROOT}
 FILES_TO_MUTATE=($(ls llvm-${LLVM_VERSION}-mutated/llvm/lib/Transforms/InstCombine/*.cpp | sort))
 echo ${FILES[*]}
@@ -94,6 +109,9 @@ ${DREDD_EXECUTABLE} -p llvm-${LLVM_VERSION}-mutated-build/compile_commands.json
 Apply mutation tracking to all `.cpp` files under `InstCombine` in the copy of LLVM designated for mutation tracking:
 
 ```
+# (Optional) `sort` depend on locale, for reproducibility:
+export LC_ALL=C
+
 cd ${DREDD_EXPERIMENTS_ROOT}
 FILES_TO_MUTATE=($(ls llvm-${LLVM_VERSION}-mutant-tracking/llvm/lib/Transforms/InstCombine/*.cpp | sort))
 echo ${FILES[*]}
@@ -203,6 +221,11 @@ cmake -S csmith -B csmith/build -G Ninja
 cmake --build csmith/build
 ```
 
+`csmith-runner` and `reduce-new-kills` both use `clang-15`'s sanitiser, which might not work on newer Linux distros. A workaround for this issue is to reduce ASLR entropy:
+```
+sudo sysctl vm.mmap_rnd_bits=28
+```
+
 ```
 csmith-runner llvm-mutated.json llvm-mutant-tracking.json llvm-${LLVM_VERSION}-mutated-build/bin/clang llvm-${LLVM_VERSION}-mutant-tracking-build/bin/clang ${DREDD_EXPERIMENTS_ROOT}/csmith
 ```
@@ -219,6 +242,31 @@ To kill them:
 pkill -9 -f csmith-runner
 ```
 
+# (or alternatively) YARPGen runner
+
+Get and build YARPGen:
+```
+git clone https://github.com/intel/yarpgen.git
+pushd yarpgen
+# (Optional) for reproducibility
+git checkout 700f5a2f564aab697ef8ff1b26afd50c3e729ecb
+
+mkdir build
+cd build
+cmake ..
+make -j$(proc)
+popd
+```
+
+```
+yarpgen-runner llvm-mutated.json llvm-mutant-tracking.json llvm-${LLVM_VERSION}-mutated-build/bin/clang llvm-${LLVM_VERSION}-mutant-tracking-build/bin/clang ${DREDD_EXPERIMENTS_ROOT}/yarpgen
+```
+
+To run many instances in parallel (16):
+
+```
+for i in `seq 1 16`; do yarpgen-runner llvm-mutated.json llvm-mutant-tracking.json llvm-${LLVM_VERSION}-mutated-build/bin/clang llvm-${LLVM_VERSION}-mutant-tracking-build/bin/clang ${DREDD_EXPERIMENTS_ROOT}/yarpgen & done
+```
 
 # Results analysis
 
@@ -230,8 +278,47 @@ analyse-results work
 ```
 
 # Reductions
+Install `creduce` and `gcc-12`:
+```
+sudo apt install creduce gcc-12
+```
 
 ```
 cd ${DREDD_EXPERIMENTS_ROOT}
 reduce-new-kills work ${DREDD_EXPERIMENTS_ROOT}/llvm-${LLVM_VERSION}-mutated-build/bin/clang ${DREDD_EXPERIMENTS_ROOT}/csmith
+```
+
+To run many instances in parallel (16):
+
+```
+for i in `seq 1 16`; do reduce-new-kills work ${DREDD_EXPERIMENTS_ROOT}/llvm-${LLVM_VERSION}-mutated-build/bin/clang ${DREDD_EXPERIMENTS_ROOT}/csmith & done
+```
+
+# Package grown testsuite
+The following runner verifies that the reduced program is compilable with Clang/GCC under both `-O3` and `-O0` optimization levels. In the case of a miscompilation test case, it checks that the execution output of the binary, compiled by each compiler and optimization mode, produces the same result.
+
+```
+package-tests work ${DREDD_EXPERIMENTS_ROOT}/csmith
+```
+
+To run many instances in parallel (16):
+
+```
+for i in `seq 1 16`; do package-tests work ${DREDD_EXPERIMENTS_ROOT}/csmith & done
+```
+
+# Historical check
+
+Make sure the following packages are installed:
+```
+sudo apt install gcc-multilib libncurses5
+```
+
+You might need to remove testcases that failed to be packaged in `package-tests`:
+```
+find work/testsuite/ -empty -type d -delete
+```
+
+```
+historical-check work ${LLVM_VERSION} ${DREDD_EXPERIMENTS_ROOT}/csmith/
 ```

dredd_test_runners/analyse_results/main.py

@@ -44,7 +44,7 @@ def main():
             elif kill_type == 'KillStatus.KILL_DIFFERENT_EXIT_CODES':
                 print(mutant_summary)
             elif kill_type == 'KillStatus.KILL_COMPILER_CRASH':
-                pass
+                print(mutant_summary)
             elif kill_type == 'KillStatus.KILL_COMPILER_TIMEOUT':
                 pass
             else:

dredd_test_runners/common/mutation_tree.py

@@ -35,7 +35,7 @@ def populate(json_node, node_id):
                 children.append(child_node_id)
                 self.parent_map[child_node_id] = node_id
                 self.num_nodes += 1
-                populate(child_json_node, child_node_id)
+                # populate(child_json_node, child_node_id)
             self.nodes[node_id] = MutationTreeNode(get_mutation_ids_for_json_node(json_node), children)
             temp: int = functools.reduce(max, self.nodes[node_id].mutation_ids, 0)
             self.num_mutations = max(self.num_mutations, temp)
@@ -48,10 +48,11 @@ def populate(json_node, node_id):
         self.num_mutations = 0
         self.num_nodes = 0
 
-        for root_json_node in [file["mutationTreeRoot"] for file in json_data["infoForFiles"]]:
-            root_node_id = self.num_nodes
-            self.num_nodes += 1
-            populate(root_json_node, root_node_id)
+        for file_info in json_data["infoForFiles"]:
+            for mutation_tree_node in file_info["mutationTree"]: 
+                root_node_id = self.num_nodes
+                self.num_nodes += 1
+                populate(mutation_tree_node, root_node_id)
 
     def get_mutation_ids_for_subtree(self, node_id) -> List[int]:
         assert 0 <= node_id < self.num_nodes

dredd_test_runners/csmith_runner/main.py

@@ -6,6 +6,7 @@
 import random
 import tempfile
 import time
+import datetime
 
 from dredd_test_runners.common.constants import DEFAULT_COMPILATION_TIMEOUT, DEFAULT_RUNTIME_TIMEOUT
 from dredd_test_runners.common.hash_file import hash_file
@@ -259,6 +260,9 @@ def main():
                 continue
             shutil.copy(src=csmith_generated_program, dst=test_output_directory / "prog.c")
 
+            # Record time at which consideration of this test started
+            analysis_timestamp_start: datetime.datetime = datetime.datetime.now()
+
             # Load file contents into a list. We go from list to set to list to eliminate duplicates.
             covered_by_this_test: List[int] = list(set([int(line.strip()) for line in
                                                         open(dredd_covered_mutants_path, 'r').readlines()]))
@@ -310,7 +314,9 @@ def main():
                     print("Writing kill info to file.")
                     with open(mutant_path / "kill_info.json", "w") as outfile:
                         json.dump({"killing_test": csmith_test_name,
-                                   "kill_type": str(mutant_result)}, outfile)
+                                   "kill_type": str(mutant_result),
+                                   "kill_timestamp": str(datetime.datetime.now()),
+                                   }, outfile)
                 except FileExistsError:
                     print(f"Mutant {mutant} was independently discovered to be killed.")
                     continue
@@ -335,12 +341,19 @@ def main():
             killed_by_this_test.sort()
             covered_but_not_killed_by_this_test.sort()
             already_killed_by_other_tests.sort()
+
+            # Record time at which consideration of this test ended
+            analysis_timestamp_end: datetime.datetime = datetime.datetime.now()
+
             with open(test_output_directory / "kill_summary.json", "w") as outfile:
                 json.dump({"terminated_early": terminated_early,
-                           "covered_mutants": covered_by_this_test,
+                           "covered_mutants_count": len(covered_by_this_test),
                            "killed_mutants": killed_by_this_test,
-                           "skipped_mutants": already_killed_by_other_tests,
-                           "survived_mutants": covered_but_not_killed_by_this_test}, outfile)
+                           "skipped_mutants_count": len(already_killed_by_other_tests),
+                           "survived_mutants_count": len(covered_but_not_killed_by_this_test),
+                           "analysis_start_time": str(analysis_timestamp_start),
+                           "analysis_end_time": str(analysis_timestamp_end),
+                           }, outfile)
 
 
 if __name__ == '__main__':

dredd_test_runners/historical_check/get_clang_llvm_releases.py

@@ -0,0 +1,100 @@
+import argparse
+import re
+import requests
+import urllib.parse
+
+from pathlib import Path
+from typing import List
+from packaging import version
+from collections import OrderedDict
+
+PRE_GITHUH_RELEASE_URLS = url_pre_github = [
+    "https://releases.llvm.org/7.0.1/clang+llvm-7.0.1-x86_64-linux-gnu-ubuntu-18.04.tar.xz",
+    "https://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/6.0.1/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/6.0.0/clang+llvm-6.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/5.0.2/clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/5.0.1/clang+llvm-5.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu16.04.tar.xz",
+    "https://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.10.tar.xz",
+    "https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/3.9.0/clang+llvm-3.9.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/3.8.1/clang+llvm-3.8.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/3.8.0/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
+    "https://releases.llvm.org/3.7.1/clang+llvm-3.7.1-x86_64-linux-gnu-ubuntu-15.10.tar.xz",
+    "https://releases.llvm.org/3.7.0/clang+llvm-3.7.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz",
+    "https://releases.llvm.org/3.6.2/clang+llvm-3.6.2-x86_64-linux-gnu-ubuntu-15.04.tar.xz",
+    "https://releases.llvm.org/3.6.1/clang+llvm-3.6.1-x86_64-linux-gnu-ubuntu-15.04.tar.xz",
+    "https://releases.llvm.org/3.6.0/clang+llvm-3.6.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz",
+    "https://releases.llvm.org/3.5.2/clang+llvm-3.5.2-x86_64-linux-gnu-ubuntu-14.04.tar.xz",
+    "https://releases.llvm.org/3.5.1/clang+llvm-3.5.1-x86_64-linux-gnu.tar.xz",
+    "https://releases.llvm.org/3.5.0/clang+llvm-3.5.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz",
+    "https://releases.llvm.org/3.4.2/clang+llvm-3.4.2-x86_64-linux-gnu-ubuntu-14.04.xz",
+    "https://releases.llvm.org/3.4.1/clang+llvm-3.4.1-x86_64-unknown-ubuntu12.04.tar.xz",
+    "https://releases.llvm.org/3.4/clang+llvm-3.4-x86_64-linux-gnu-ubuntu-13.10.tar.xz",
+    "https://releases.llvm.org/3.3/clang+llvm-3.3-Ubuntu-13.04-x86_64-linux-gnu.tar.bz2",
+    "https://releases.llvm.org/3.2/clang+llvm-3.2-x86_64-linux-ubuntu-12.04.tar.gz",
+    "https://releases.llvm.org/3.1/clang+llvm-3.1-x86_64-linux-ubuntu_12.04.tar.gz",
+    "https://releases.llvm.org/3.0/clang+llvm-3.0-x86_64-linux-Ubuntu-11_10.tar.gz",
+    "https://releases.llvm.org/2.9/clang+llvm-2.9-x86_64-linux.tar.bz2",
+    "https://releases.llvm.org/2.8/clang+llvm-2.8-x86_64-linux.tar.bz2",
+    "https://releases.llvm.org/2.7/clang+llvm-2.7-x86_64-linux.tar.bz2",
+    "https://releases.llvm.org/2.6/llvm+clang-2.6-x86_64-linux.tar.gz"
+]
+
+def get_clang_llvm_releases(after_version: str) -> List[str]:
+    result : List[str] = []
+
+    page = 0
+    ubuntu_release_pattern = r"^clang\+llvm-(\d+\.\d+\.\d+)-x86_64-linux-gnu-ubuntu-(\d+\.\d+)\.tar\.xz$"
+    while True:
+        response = requests.get(f'https://api.github.com/repos/llvm/llvm-project/releases?page={page}')
+        if len(response.json()) == 0:
+            break
+        if response.status_code != 200:
+            raise Exception(response.json()['message']) 
+
+        for release in response.json():
+            # Skip Pre-release version
+            if release['prerelease']:
+                continue
+
+            # Get release for latest ubuntu version
+            latest_ubuntu_release = ""
+            latest_ubuntu_version = ""
+            for asset in release['assets']:
+                url = urllib.parse.unquote(asset['browser_download_url'])
+                tar_file = url.split('/')[-1]
+                match = re.match(ubuntu_release_pattern, tar_file)
+                if not match:
+                    continue
+
+                # Version smaller than requested versions, return result
+                if version.parse(match.group(1)) <= version.parse(after_version):
+                    return list(reversed(OrderedDict.fromkeys(result)))
+
+                if latest_ubuntu_version == "" or version.parse(match.group(2)) > version.parse(latest_ubuntu_version):
+                    latest_ubuntu_version = match.group(2)
+                    latest_ubuntu_release = url
+            if latest_ubuntu_release != "":
+                result.append(latest_ubuntu_release)
+        page += 1
+
+    # Continue searching for releases from `PRE_GITHUH_RELEASE_URLS`
+    for release_url in PRE_GITHUH_RELEASE_URLS:
+        release_version = release_url.replace("https://releases.llvm.org/", '').split('/')[0]
+
+        if version.parse(release_version) <= version.parse(after_version):
+            break 
+        result.append(release_url)
+
+    return list(reversed(OrderedDict.fromkeys(result)))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("after_version", help="Get list of llvm release after this version", type=str)
+    args = parser.parse_args()
+    release_urls = get_clang_llvm_releases(args.after_version)
+    for url in release_urls:
+        print(url)