Add bignum test case generation script

scripts/mbedtls_dev/test_generation.py

@@ -35,14 +35,16 @@
 class BaseTarget(metaclass=ABCMeta):
     """Base target for test case generation.
 
-    Derive directly from this class when adding new file Targets, setting
-    `target_basename`.
+    Child classes of this class represent an output file, and can be referred
+    to as file targets. These indicate where test cases will be written to for
+    all subclasses of the file target, which is set by `target_basename`.
 
     Attributes:
         count: Counter for test cases from this class.
         case_description: Short description of the test case. This may be
             automatically generated using the class, or manually set.
         dependencies: A list of dependencies required for the test case.
+        show_test_count: Toggle for inclusion of `count` in the test description.
         target_basename: Basename of file to write generated tests to. This
             should be specified in a child class of BaseTarget.
         test_function: Test function which the class generates cases for.
@@ -53,6 +55,7 @@ class BaseTarget(metaclass=ABCMeta):
     count = 0
     case_description = ""
     dependencies = [] # type: List[str]
+    show_test_count = True
     target_basename = ""
     test_function = ""
     test_name = ""
@@ -78,16 +81,19 @@ def description(self) -> str:
         """Create a test case description.
 
         Creates a description of the test case, including a name for the test
-        function, a case number, and a description the specific test case.
-        This should inform a reader what is being tested, and provide context
-        for the test case.
+        function, an optional case count, and a description of the specific
+        test case. This should inform a reader what is being tested, and
+        provide context for the test case.
 
         Returns:
             Description for the test case.
         """
-        return "{} #{} {}".format(
-            self.test_name, self.count, self.case_description
-            ).strip()
+        if self.show_test_count:
+            return "{} #{} {}".format(
+                self.test_name, self.count, self.case_description
+                ).strip()
+        else:
+            return "{} {}".format(self.test_name, self.case_description).strip()
 
 
     def create_test_case(self) -> test_case.TestCase:
@@ -130,15 +136,23 @@ def generate_tests(cls) -> Iterator[test_case.TestCase]:
 
 
 class TestGenerator:
-    """Generate test data."""
+    """Generate test cases and write to data files."""
     def __init__(self, options) -> None:
-        self.test_suite_directory = getattr(options, 'directory')
-        # Add file Targets which have been declared in other modules
+        self.test_suite_directory = self.get_option(options, 'directory',
+                                                    'tests/suites')
+        # Update `targets` with an entry for each child class of BaseTarget.
+        # Each entry represents a file generated by the BaseTarget framework,
+        # and enables generating the .data files using the CLI.
         self.targets.update({
             subclass.target_basename: subclass.generate_tests
             for subclass in BaseTarget.__subclasses__()
         })
 
+    @staticmethod
+    def get_option(options, name: str, default: T) -> T:
+        value = getattr(options, name, None)
+        return default if value is None else value
+
     def filename_for(self, basename: str) -> str:
         """The location of the data file with the specified base name."""
         return posixpath.join(self.test_suite_directory, basename + '.data')
@@ -165,15 +179,19 @@ def generate_target(self, name: str, *target_args) -> None:
         test_cases = self.targets[name](*target_args)
         self.write_test_data_file(name, test_cases)
 
-def main(args, generator_class: Type[TestGenerator] = TestGenerator):
+def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator):
     """Command line entry point."""
-    parser = argparse.ArgumentParser(description=__doc__)
+    parser = argparse.ArgumentParser(description=description)
     parser.add_argument('--list', action='store_true',
                         help='List available targets and exit')
     parser.add_argument('--list-for-cmake', action='store_true',
                         help='Print \';\'-separated list of available targets and exit')
-    parser.add_argument('--directory', default="tests/suites", metavar='DIR',
+    parser.add_argument('--directory', metavar='DIR',
                         help='Output directory (default: tests/suites)')
+    # The `--directory` option is interpreted relative to the directory from
+    # which the script is invoked, but the default is relative to the root of
+    # the mbedtls tree. The default should not be set above, but instead after
+    # `build_tree.chdir_to_root()` is called.
     parser.add_argument('targets', nargs='*', metavar='TARGET',
                         help='Target file to generate (default: all; "-": none)')
     options = parser.parse_args(args)

tests/scripts/generate_bignum_tests.py

@@ -6,11 +6,11 @@
 
 Class structure:
 
-Child classes of test_generation.BaseTarget (file Targets) represent a target
+Child classes of test_generation.BaseTarget (file targets) represent an output
 file. These indicate where test cases will be written to, for all subclasses of
-this Target. Multiple Target classes should not reuse a `target_basename`.
+this target. Multiple file targets should not reuse a `target_basename`.
 
-Each subclass derived from a file Target can either be:
+Each subclass derived from a file target can either be:
   - A concrete class, representing a test function, which generates test cases.
   - An abstract class containing shared methods and attributes, not associated
         with a test function. An example is BignumOperation, which provides
@@ -24,7 +24,7 @@
 Adding test case generation for a function:
 
 A subclass representing the test function should be added, deriving from a
-file Target such as BignumTarget. This test class must set/implement the
+file target such as BignumTarget. This test class must set/implement the
 following:
   - test_function: the function name from the associated .function file.
   - test_name: a descriptive name or brief summary to refer to the test
@@ -56,9 +56,10 @@
 
 import itertools
 import sys
+import typing
 
 from abc import ABCMeta, abstractmethod
-from typing import Iterator, List, Tuple, TypeVar, cast
+from typing import Iterator, List, Tuple, TypeVar
 
 import scripts_path # pylint: disable=unused-import
 from mbedtls_dev import test_case
@@ -72,6 +73,17 @@ def hex_to_int(val: str) -> int:
 def quote_str(val) -> str:
     return "\"{}\"".format(val)
 
+def combination_pairs(values: List[T]) -> List[Tuple[T, T]]:
+    """Return all pair combinations from input values.
+
+    The return value is cast, as older versions of mypy are unable to derive
+    the specific type returned by itertools.combinations_with_replacement.
+    """
+    return typing.cast(
+        List[Tuple[T, T]],
+        list(itertools.combinations_with_replacement(values, 2))
+    )
+
 
 class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta):
     #pylint: disable=abstract-method
@@ -99,7 +111,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta):
         "0000000000000000123", "-0000000000000000123",
         "1230000000000000000", "-1230000000000000000"
     ] # type: List[str]
-    input_cases = cast(List[Tuple[str, str]], []) # type: List[Tuple[str, str]]
+    input_cases = [] # type: List[Tuple[str, str]]
 
     def __init__(self, val_a: str, val_b: str) -> None:
         self.arg_a = val_a
@@ -164,10 +176,7 @@ def get_value_pairs(cls) -> Iterator[Tuple[str, str]]:
         Combinations are first generated from all input values, and then
         specific cases provided.
         """
-        yield from cast(
-            Iterator[Tuple[str, str]],
-            itertools.combinations_with_replacement(cls.input_values, 2)
-        )
+        yield from combination_pairs(cls.input_values)
         yield from cls.input_cases
 
     @classmethod
@@ -214,19 +223,16 @@ class BignumAdd(BignumOperation):
     symbol = "+"
     test_function = "mbedtls_mpi_add_mpi"
     test_name = "MPI add"
-    input_cases = cast(
-        List[Tuple[str, str]],
-        list(itertools.combinations_with_replacement(
-            [
-                "1c67967269c6", "9cde3",
-                "-1c67967269c6", "-9cde3",
-            ], 2
-        ))
+    input_cases = combination_pairs(
+        [
+            "1c67967269c6", "9cde3",
+            "-1c67967269c6", "-9cde3",
+        ]
     )
 
     def result(self) -> str:
         return quote_str("{:x}".format(self.int_a + self.int_b))
 
-
 if __name__ == '__main__':
-    test_generation.main(sys.argv[1:])
+    # Use the section of the docstring relevant to the CLI as description
+    test_generation.main(sys.argv[1:], "\n".join(__doc__.splitlines()[:4]))

tests/scripts/generate_psa_tests.py

@@ -917,4 +917,4 @@ def generate_target(self, name: str, *target_args) -> None:
         super().generate_target(name, self.info)
 
 if __name__ == '__main__':
-    test_generation.main(sys.argv[1:], PSATestGenerator)
+    test_generation.main(sys.argv[1:], __doc__, PSATestGenerator)