Define interface for Acto's test results (#303)

* Define interface for test result Signed-off-by: Tyler Gu <[email protected]> Fix run_trial return stmt Signed-off-by: Tyler Gu <[email protected]> Fix results' serialization Signed-off-by: Tyler Gu <[email protected]> Fix bug reproduction test Signed-off-by: Tyler Gu <[email protected]> Fix Secret number comparison Signed-off-by: Tyler Gu <[email protected]> * Fix model serialization Signed-off-by: Tyler Gu <[email protected]> * Fix TrialResult dump Signed-off-by: Tyler Gu <[email protected]> --------- Signed-off-by: Tyler Gu <[email protected]>
xlab-uiuc · Jan 15, 2024 · aeca446 · aeca446
1 parent 8b2e02a
commit aeca446
Show file tree

Hide file tree

Showing 288 changed files with 196,041 additions and 199,338 deletions.
diff --git a/acto/checker/checker.py b/acto/checker/checker.py
@@ -1,19 +1,20 @@
+"""Checker interface"""
 from abc import ABC, abstractmethod
+from typing import Optional
 
-from acto.common import OracleResult
+from acto.result import OracleResult
 from acto.snapshot import Snapshot
 
 
-class Checker(ABC):
+class CheckerInterface(ABC):
+    """Interface for checkers"""
 
-    @property
-    @abstractmethod
-    def name(self):
-        raise NotImplementedError
-
-    def __init__(self, **kwargs):
+    def __init__(self):
         pass
 
     @abstractmethod
-    def check(self, generation: int, snapshot: Snapshot, prev_snapshot: Snapshot) -> OracleResult:
+    def check(
+        self, generation: int, snapshot: Snapshot, prev_snapshot: Snapshot
+    ) -> Optional[OracleResult]:
+        """Check the given step and return the result of the check"""
         raise NotImplementedError()
diff --git a/acto/checker/checker_set.py b/acto/checker/checker_set.py
@@ -1,18 +1,15 @@
 """Oracle checker set of Acto."""
 
-import json
-import os
 from typing import Optional
 
+from acto.checker.impl.consistency import ConsistencyChecker
 from acto.checker.impl.crash import CrashChecker
 from acto.checker.impl.health import HealthChecker
-from acto.checker.impl.kubectl_cli import KubectlCliChecker
 from acto.checker.impl.operator_log import OperatorLogChecker
-from acto.checker.impl.state import StateChecker
-from acto.common import InvalidInputResult, RunResult, flatten_dict
+from acto.common import flatten_dict
 from acto.input import InputModel
 from acto.oracle_handle import OracleHandle
-from acto.serialization import ActoEncoder
+from acto.result import OracleResults
 from acto.snapshot import Snapshot
 from acto.utils import get_thread_logger
 
@@ -28,50 +25,34 @@ def __init__(
         oracle_handle: OracleHandle,
         checker_generators: Optional[list] = None,
     ):
-        checker_generators = [
-            CrashChecker,
-            HealthChecker,
-            KubectlCliChecker,
-            OperatorLogChecker,
-            StateChecker,
-        ]
         if checker_generators:
             checker_generators.extend(checker_generators)
         self.context = context
         self.input_model = input_model
         self.trial_dir = trial_dir
 
-        checker_args = {
-            "trial_dir": self.trial_dir,
-            "input_model": self.input_model,
-            "context": self.context,
-            "oracle_handle": oracle_handle,
-        }
-        self.checkers = [
-            checkerGenerator(**checker_args)
-            for checkerGenerator in checker_generators
-        ]
+        self._crash_checker = CrashChecker()
+        self._health_checker = HealthChecker()
+        self._operator_log_checker = OperatorLogChecker()
+        self._consistency_checker = ConsistencyChecker(
+            trial_dir=self.trial_dir,
+            context=self.context,
+            input_model=self.input_model,
+        )
+        _ = oracle_handle
 
     def check(
         self,
         snapshot: Snapshot,
         prev_snapshot: Snapshot,
-        revert: bool,
         generation: int,
-        testcase_signature: dict,
-    ) -> RunResult:
+    ) -> OracleResults:
         """Run all checkers on the given snapshot and return the result."""
         logger = get_thread_logger(with_prefix=True)
 
-        run_result = RunResult(revert, generation, testcase_signature)
-
         if snapshot.system_state == {}:
-            run_result.misc_result = InvalidInputResult([])
-            return run_result
-
-        for checker in self.checkers:
-            checker_result = checker.check(generation, snapshot, prev_snapshot)
-            run_result.set_result(checker.name, checker_result)
+            logger.warning("System state is empty, skipping check")
+            return OracleResults()
 
         input_delta, system_delta = snapshot.delta(prev_snapshot)
         flattened_system_state = flatten_dict(snapshot.system_state, [])
@@ -87,13 +68,26 @@ def check(
                 num_delta,
             )
 
-        generation_result_path = os.path.join(
-            self.trial_dir, f"generation-{generation:03d}-runtime.json"
+        # generation_result_path = os.path.join(
+        #     self.trial_dir, f"generation-{generation:03d}-runtime.json"
+        # )
+        # with open(generation_result_path, "w", encoding="utf-8") as f:
+        #     json.dump(run_result.to_dict(), f, cls=ActoEncoder, indent=4)
+
+        return OracleResults(
+            crash=self._crash_checker.check(
+                generation, snapshot, prev_snapshot
+            ),
+            health=self._health_checker.check(
+                generation, snapshot, prev_snapshot
+            ),
+            operator_log=self._operator_log_checker.check(
+                generation, snapshot, prev_snapshot
+            ),
+            consistency=self._consistency_checker.check(
+                generation, snapshot, prev_snapshot
+            ),
         )
-        with open(generation_result_path, "w", encoding="utf-8") as f:
-            json.dump(run_result.to_dict(), f, cls=ActoEncoder, indent=4)
-
-        return run_result
 
     def count_num_fields(self, snapshot: Snapshot, prev_snapshot: Snapshot):
         """Count the number of fields in the system state and the number of delta."""