Merge conflict

README.md

@@ -158,6 +158,14 @@ python3 acto.py --config data/rabbitmq-operator/config.json
                 --num-workers 4
 ```
 
+## Reproduce previously found bugs
+Reproduction utility enables Acto to reproduce previously found bugs by taking a folder that contains previously generated CRs (i.e. mutated files) as input and directly deploying each CR. In this way, to reproduce a bug in a certain operator, Acto can just run a single testcase instead of running all testcases for that operator.
+
+Usage:
+```console
+python3 reproduce.py --reproduce-dir <path to the folder containing CRs> --config <path to corresponding config.json>
+```
+
 ## Porting operators
 Acto aims to automate the E2E testing as much as possible to minimize users' labor.
 

acto.py

@@ -3,7 +3,7 @@
 import os
 import sys
 import threading
-from types import SimpleNamespace
+from types import SimpleNamespace, FunctionType
 import yaml
 import time
 from typing import List, Tuple
@@ -115,7 +115,8 @@ class TrialRunner:
 
     def __init__(self, context: dict, input_model: InputModel, deploy: Deploy,
                  custom_oracle: List[callable], workdir: str, cluster: base.KubernetesCluster,
-                 worker_id: int, dryrun: bool) -> None:
+                 worker_id: int, dryrun: bool, is_reproduce: bool,
+                 apply_testcase_f: FunctionType) -> None:
         self.context = context
         self.workdir = workdir
         self.cluster = cluster
@@ -129,9 +130,11 @@ def __init__(self, context: dict, input_model: InputModel, deploy: Deploy,
 
         self.custom_oracle = custom_oracle
         self.dryrun = dryrun
+        self.is_reproduce = is_reproduce
 
         self.snapshots = []
         self.discarded_testcases = {}  # List of test cases failed to run
+        self.apply_testcase_f = apply_testcase_f
 
     def run(self):
         logger = get_thread_logger(with_prefix=False)
@@ -190,8 +193,8 @@ def run_trial(self,
         '''
 
         oracle_handle = OracleHandle(KubectlClient(self.kubeconfig, self.context_name),
-                                      kubernetes_client(self.kubeconfig, self.context_name),
-                                      self.context['namespace'])
+                                     kubernetes_client(self.kubeconfig, self.context_name),
+                                     self.context['namespace'])
         custom_oracle = [functools.partial(i, oracle_handle) for i in self.custom_oracle]
         runner = Runner(self.context, trial_dir, self.kubeconfig, self.context_name)
         checker = Checker(self.context, trial_dir, self.input_model, custom_oracle)
@@ -226,10 +229,11 @@ def run_trial(self,
                         # precondition fails, first run setup
                         logger.info('Precondition of %s fails, try setup first',
                                     field_node.get_path())
-                        apply_testcase(curr_input_with_schema,
-                                       field_node.get_path(),
-                                       testcase,
-                                       setup=True)
+
+                        self.apply_testcase_f(curr_input_with_schema,
+                                              field_node.get_path(),
+                                              testcase,
+                                              setup=True)
 
                         if not testcase.test_precondition(
                                 curr_input_with_schema.get_value_by_path(list(
@@ -300,7 +304,8 @@ def run_testcases(self, curr_input_with_schema, testcases, runner, checker,
 
         testcase_patches = []
         for field_node, testcase in testcases:
-            patch = apply_testcase(curr_input_with_schema, field_node.get_path(), testcase)
+            patch = self.apply_testcase_f(curr_input_with_schema, field_node.get_path(), testcase)
+
             # field_node.get_testcases().pop()  # finish testcase
             testcase_patches.append((field_node, testcase, patch))
 
@@ -369,8 +374,9 @@ def run_testcases(self, curr_input_with_schema, testcases, runner, checker,
                         return self.run_testcases(curr_input_with_schema, ready_testcases, runner,
                                                   checker, generation)
         else:
-            for patch in testcase_patches:
-                patch[0].get_testcases().pop()  # finish testcase
+            if not self.is_reproduce:
+                for patch in testcase_patches:
+                    patch[0].get_testcases().pop()  # finish testcase
             if isinstance(result, UnchangedInputResult):
                 pass
             elif isinstance(result, ErrorResult):
@@ -438,6 +444,10 @@ def __init__(self,
                  num_cases: int,
                  dryrun: bool,
                  analysis_only: bool,
+                 is_reproduce: bool,
+                 input_model,
+                 apply_testcase_f: FunctionType,
+                 reproduce_dir: str = None,
                  mount: list = None) -> None:
         logger = get_thread_logger(with_prefix=False)
 
@@ -477,6 +487,10 @@ def __init__(self,
         self.images_archive = os.path.join(workdir_path, 'images.tar')
         self.num_workers = num_workers
         self.dryrun = dryrun
+        self.is_reproduce = is_reproduce
+        self.input_model = input_model
+        self.apply_testcase_f = apply_testcase_f
+        self.reproduce_dir = reproduce_dir
         self.snapshots = []
 
         # generate configuration files for the cluster runtime
@@ -491,8 +505,8 @@ def __init__(self,
             self.context['preload_images'].update(preload_images_)
 
         # Apply custom fields
-        self.input_model = InputModel(self.context['crd']['body'], operator_config.example_dir,
-                                      num_workers, num_cases, mount)
+        self.input_model = input_model(self.context['crd']['body'], operator_config.example_dir,
+                                       num_workers, num_cases, self.reproduce_dir, mount)
         self.input_model.initialize(self.seed)
         if operator_config.custom_fields != None:
             pruned_list = []
@@ -599,7 +613,8 @@ def run(self):
         threads = []
         for i in range(self.num_workers):
             runner = TrialRunner(self.context, self.input_model, self.deploy, self.custom_oracle,
-                                 self.workdir_path, self.cluster, i, self.dryrun)
+                                 self.workdir_path, self.cluster, i, self.dryrun, self.is_reproduce,
+                                 self.apply_testcase_f)
             t = threading.Thread(target=runner.run, args=())
             t.start()
             threads.append(t)
@@ -750,10 +765,18 @@ def thread_excepthook(args):
     else:
         context_cache = args.context
 
+    # Initialize input model and the apply testcase function
+    # input_model = InputModel(context_cache['crd']['body'], config.example_dir,
+    #   args.num_workers, args.num_cases, None)
+    input_model = InputModel
+    apply_testcase_f = apply_testcase
+    is_reproduce = False
+
     start_time = datetime.now()
     acto = Acto(workdir_path, config, args.cluster_runtime, args.enable_analysis,
                 args.preload_images, context_cache, args.helper_crd, args.num_workers,
-                args.num_cases, args.dryrun, args.learn_analysis_only)
+                args.num_cases, args.dryrun, args.learn_analysis_only, is_reproduce, input_model,
+                apply_testcase_f)
     if not args.learn:
         acto.run()
     end_time = datetime.now()

input.py

@@ -104,11 +104,11 @@ def __init__(self, path, array: bool = False) -> None:
             super().__init__(path, self.PruneEntireArraySchema)
         else:
             super().__init__(path, self.PruneEntireObjectSchema)
-
+            
 
 class InputModel:
 
-    def __init__(self, crd: dict, example_dir: str, num_workers: int, num_cases: int, mount: list = None) -> None:
+    def __init__(self, crd: dict, example_dir: str, num_workers: int, num_cases: int, reproduce_dir: str, mount: list = None) -> None:
         if mount != None:
             self.mount = mount
         else:
@@ -171,7 +171,7 @@ def get_root_schema(self) -> BaseSchema:
     def get_discarded_tests(self) -> dict:
         return self.discarded_tests
 
-    def generate_test_plan(self):
+    def generate_test_plan(self) -> dict:
         '''Generate test plan based on CRD'''
         logger = get_thread_logger(with_prefix=False)