Support using examples when executing tests

Signed-off-by: Tyler Gu <[email protected]>

acto/input/test_generators/primitive.py

@@ -142,7 +142,7 @@ def empty_mutator(prev):
         return []
 
     def empty_setup(prev):
-        return prev
+        return schema.gen(size=1)
 
     def delete(prev):
         return schema.empty_value()
@@ -579,6 +579,7 @@ def object_tests(schema: ObjectSchema):
 
     DELETION_TEST = "object-deletion"
     EMPTY_TEST = "object-empty"
+    CHANGE_TEST = "object-change"
 
     def empty_precondition(prev):
         return prev != {}
@@ -587,7 +588,7 @@ def empty_mutator(prev):
         return {}
 
     def empty_setup(prev):
-        return prev
+        return schema.gen(exclude_value=schema.default)
 
     def delete(prev):
         return schema.empty_value()
@@ -617,6 +618,12 @@ def delete_setup(prev):
         else:
             return schema.gen(exclude_value=schema.default)
 
+    def change_precondition(prev):
+        return prev is not None
+
+    def change(prev):
+        return prev
+
     ret = [
         TestCase(
             DELETION_TEST,
@@ -643,9 +650,36 @@ def delete_setup(prev):
 
 
 @test_generator(property_type="Opaque", priority=Priority.PRIMITIVE)
-def opaque_gen(schema: OpaqueSchema):
-    """Opaque schema to handle the fields that do not have a schema"""
-    return []
+def opaque_tests(schema: OpaqueSchema):
+    """Opaque schema to handle the fields that do not have a schema
+
+    It only generates testcases if there are examples provided
+    """
+    DELETION_TEST = "opaque-deletion"
+    CHANGE_TEST = "opaque-change"
+    ret = []
+    if schema.examples is not None and len(schema.examples) > 0:
+        ret.append(
+            TestCase(
+                DELETION_TEST,
+                lambda prev: prev is not None,
+                lambda prev: None,
+                lambda prev: schema.examples[0],
+                primitive=True,
+            )
+        )
+
+        if len(schema.examples) > 1:
+            ret.append(
+                TestCase(
+                    CHANGE_TEST,
+                    lambda prev: prev is not None,
+                    lambda prev: schema.examples[1],
+                    lambda prev: schema.examples[0],
+                    primitive=True,
+                )
+            )
+    return ret
 
 
 @test_generator(property_type="String", priority=Priority.PRIMITIVE)
@@ -669,9 +703,11 @@ def change(prev):
         """Test case to change the value to another one"""
         logger = get_thread_logger(with_prefix=True)
         if schema.enum is not None:
-            logger.fatal(
+            logger.critical(
                 "String field with enum should not call change to mutate"
             )
+        if schema.examples is not None and len(schema.examples) > 0:
+            return schema.gen(exclude_value=prev)
         if schema.pattern is not None:
             new_string = exrex.getone(schema.pattern, schema.max_length)
         else:
@@ -701,7 +737,7 @@ def empty_mutator(prev):
         return ""
 
     def empty_setup(prev):
-        return prev
+        return schema.gen(exclude_value=schema.default)
 
     def delete(prev):
         return schema.empty_value()

acto/schema/array.py

@@ -112,25 +112,33 @@ def empty_value(self):
         return []
 
     def gen(self, exclude_value=None, minimum: bool = False, **kwargs) -> list:
+        if "size" in kwargs and kwargs["size"] is not None:
+            num = kwargs["size"]
+
         if self.enum is not None:
             if exclude_value is not None:
                 return random.choice(
                     [x for x in self.enum if x != exclude_value]
                 )
             else:
                 return random.choice(self.enum)
+
+        if self.examples and len(self.examples) > 0:
+            candidates = [
+                x for x in self.examples if x != exclude_value and len(x) > num
+            ]
+            if candidates:
+                return random.choice(candidates)[num:]
+
+        # XXX: need to handle exclude_value, but not important for now for array types
+        result = []
+        if minimum:
+            num = self.min_items
         else:
-            # XXX: need to handle exclude_value, but not important for now for array types
-            result = []
-            if "size" in kwargs and kwargs["size"] is not None:
-                num = kwargs["size"]
-            elif minimum:
-                num = self.min_items
-            else:
-                num = random.randint(self.min_items, self.max_items)
-            for _ in range(num):
-                result.append(self.item_schema.gen(minimum=minimum))
-            return result
+            num = random.randint(self.min_items, self.max_items)
+        for _ in range(num):
+            result.append(self.item_schema.gen(minimum=minimum))
+        return result
 
     def __str__(self) -> str:
         return "Array"

acto/schema/object.py

@@ -173,6 +173,16 @@ def gen(self, exclude_value=None, minimum: bool = False, **kwargs):
             else:
                 return random.choice(self.enum)
 
+        if self.examples:
+            if exclude_value is not None:
+                example_without_exclude = [
+                    x for x in self.examples if x != exclude_value
+                ]
+                if example_without_exclude:
+                    return random.choice(example_without_exclude)
+            else:
+                return random.choice(self.examples)
+
         # XXX: need to handle exclude_value, but not important for now for object types
         result = {}
         if len(self.properties) == 0:

acto/schema/opaque.py

@@ -17,8 +17,8 @@ def get_normal_semantic_schemas(
     def to_tree(self) -> TreeNode:
         return TreeNode(self.path)
 
-    def load_examples(self, example):
-        pass
+    def load_examples(self, example: object):
+        self.examples.append(example)
 
     def set_default(self, instance):
         self.default = instance

acto/schema/string.py

@@ -1,5 +1,5 @@
 import random
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 import exrex
 
@@ -55,7 +55,12 @@ def set_default(self, instance):
     def empty_value(self):
         return ""
 
-    def gen(self, exclude_value=None, minimum: bool = False, **kwargs):
+    def gen(
+        self,
+        exclude_value: Optional[str] = None,
+        minimum: bool = False,
+        **kwargs,
+    ):
         # TODO: Use minLength: the exrex does not support minLength
         if self.enum is not None:
             if exclude_value is not None:
@@ -64,12 +69,19 @@ def gen(self, exclude_value=None, minimum: bool = False, **kwargs):
                 )
             else:
                 return random.choice(self.enum)
+        if self.examples:
+            if exclude_value is not None:
+                example_without_exclude = [
+                    x for x in self.examples if x != exclude_value
+                ]
+                if len(example_without_exclude) > 0:
+                    return random.choice(example_without_exclude)
         if self.pattern is not None:
-            # XXX: since it's random, we don't need to exclude the value
+            # Since it's random, we don't need to exclude the value
             return exrex.getone(self.pattern, self.max_length)
         if minimum:
             return random_string(self.min_length)  # type: ignore
-        return "ACTOKEY"
+        return "ACTOSTRING"
 
     def __str__(self) -> str:
         return "String"