From 545911f751d972ef6b0b0634d5a1ecedb845ebfb Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 8 Jul 2022 13:54:57 +0100 Subject: [PATCH 01/42] Add bignum test generation framework Adds python script for generation of bignum test cases, with initial classes for mpi_cmp_mpi test cases. Build scripts are updated to generate test data. Signed-off-by: Werner Lewis --- tests/scripts/check-generated-files.sh | 1 + tests/scripts/generate_bignum_tests.py | 269 +++++++++++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100755 tests/scripts/generate_bignum_tests.py diff --git a/tests/scripts/check-generated-files.sh b/tests/scripts/check-generated-files.sh index 75730ba25483..3e5cb86c9890 100755 --- a/tests/scripts/check-generated-files.sh +++ b/tests/scripts/check-generated-files.sh @@ -107,4 +107,5 @@ check scripts/generate_query_config.pl programs/test/query_config.c check scripts/generate_features.pl library/version_features.c check scripts/generate_visualc_files.pl visualc/VS2010 check scripts/generate_psa_constants.py programs/psa/psa_constant_names_generated.c +check tests/scripts/generate_bignum_tests.py $(tests/scripts/generate_bignum_tests.py --list) check tests/scripts/generate_psa_tests.py $(tests/scripts/generate_psa_tests.py --list) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py new file mode 100755 index 000000000000..c6e6a116caf7 --- /dev/null +++ b/tests/scripts/generate_bignum_tests.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Generate test data for bignum functions. + +With no arguments, generate all test data. With non-option arguments, +generate only the specified files. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import itertools +import os +import posixpath +import re +import sys +from typing import Iterable, Iterator, Optional, Tuple, TypeVar + +import scripts_path # pylint: disable=unused-import +from mbedtls_dev import build_tree +from mbedtls_dev import test_case + +T = TypeVar('T') #pylint: disable=invalid-name + +def hex_to_int(val): + return int(val, 16) if val else 0 + +def quote_str(val): + return "\"{}\"".format(val) + + +class BaseTarget: + """Base target for test case generation. + + Attributes: + count: Counter for test class. + desc: Short description of test case. + func: Function which the class generates tests for. + gen_file: File to write generated tests to. + title: Description of the test function/purpose. + """ + count = 0 + desc = None + func = None + gen_file = "" + title = None + + def __init__(self) -> None: + type(self).count += 1 + + @property + def args(self) -> Iterable[str]: + """Create list of arguments for test case.""" + return [] + + @property + def description(self) -> str: + """Create a numbered test description.""" + return "{} #{} {}".format(self.title, self.count, self.desc) + + def create_test_case(self) -> test_case.TestCase: + """Generate test case from the current object.""" + tc = test_case.TestCase() + tc.set_description(self.description) + tc.set_function(self.func) + tc.set_arguments(self.args) + + return tc + + @classmethod + def generate_tests(cls): + """Generate test cases for the target subclasses.""" + for subclass in cls.__subclasses__(): + yield from subclass.generate_tests() + + +class BignumTarget(BaseTarget): + """Target for bignum (mpi) test case generation.""" + gen_file = 'test_suite_mpi.generated' + + +class BignumOperation(BignumTarget): + """Common features for test cases covering bignum operations. + + Attributes: + symb: Symbol used for operation in description. + input_vals: List of values used to generate test case args. + input_cases: List of tuples containing test case inputs. This + can be used to implement specific pairs of inputs. + """ + symb = "" + input_vals = [ + "", "0", "7b", "-7b", + "0000000000000000123", "-0000000000000000123", + "1230000000000000000", "-1230000000000000000" + ] + input_cases = [] + + def __init__(self, val_l: str, val_r: str) -> None: + super().__init__() + + self.arg_l = val_l + self.arg_r = val_r + self.int_l = hex_to_int(val_l) + self.int_r = hex_to_int(val_r) + + @property + def args(self): + return [quote_str(self.arg_l), quote_str(self.arg_r), self.result] + + @property + def description(self): + desc = self.desc if self.desc else "{} {} {}".format( + self.val_desc(self.arg_l), + self.symb, + self.val_desc(self.arg_r) + ) + return "{} #{} {}".format(self.title, self.count, desc) + + @property + def result(self) -> Optional[str]: + return None + + @staticmethod + def val_desc(val) -> str: + """Generate description of the argument val.""" + if val == "": + return "0 (null)" + if val == "0": + return "0 (1 limb)" + + if val[0] == "-": + tmp = "negative" + val = val[1:] + else: + tmp = "positive" + if val[0] == "0": + tmp += " with leading zero limb" + elif len(val) > 10: + tmp = "large " + tmp + return tmp + + @classmethod + def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: + """Generate value pairs.""" + for pair in set( + list(itertools.combinations(cls.input_vals, 2)) + + cls.input_cases + ): + yield pair + + @classmethod + def generate_tests(cls) -> Iterator[test_case.TestCase]: + if cls.func is not None: + # Generate tests for the current class + for l_value, r_value in cls.get_value_pairs(): + cur_op = cls(l_value, r_value) + yield cur_op.create_test_case() + # Once current class completed, check descendants + yield from super().generate_tests() + + +class BignumCmp(BignumOperation): + """Target for bignum comparison test cases.""" + count = 0 + func = "mbedtls_mpi_cmp_mpi" + title = "MPI compare" + input_cases = [ + ("-2", "-3"), + ("-2", "-2"), + ("2b4", "2b5"), + ("2b5", "2b6") + ] + + def __init__(self, val_l, val_r): + super().__init__(val_l, val_r) + self._result = (self.int_l > self.int_r) - (self.int_l < self.int_r) + self.symb = ["<", "==", ">"][self._result + 1] + + @property + def result(self): + return str(self._result) + + +class TestGenerator: + """Generate test data.""" + + def __init__(self, options) -> None: + self.test_suite_directory = self.get_option(options, 'directory', + 'tests/suites') + + @staticmethod + def get_option(options, name: str, default: T) -> T: + value = getattr(options, name, None) + return default if value is None else value + + def filename_for(self, basename: str) -> str: + """The location of the data file with the specified base name.""" + return posixpath.join(self.test_suite_directory, basename + '.data') + + def write_test_data_file(self, basename: str, + test_cases: Iterable[test_case.TestCase]) -> None: + """Write the test cases to a .data file. + + The output file is ``basename + '.data'`` in the test suite directory. + """ + filename = self.filename_for(basename) + test_case.write_data_file(filename, test_cases) + + # Note that targets whose names contain 'test_format' have their content + # validated by `abi_check.py`. + TARGETS = { + subclass.gen_file: subclass.generate_tests for subclass in + BaseTarget.__subclasses__() + } + + def generate_target(self, name: str) -> None: + test_cases = self.TARGETS[name]() + self.write_test_data_file(name, test_cases) + +def main(args): + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--list', action='store_true', + help='List available targets and exit') + parser.add_argument('--list-for-cmake', action='store_true', + help='Print \';\'-separated list of available targets and exit') + parser.add_argument('--directory', metavar='DIR', + help='Output directory (default: tests/suites)') + parser.add_argument('targets', nargs='*', metavar='TARGET', + help='Target file to generate (default: all; "-": none)') + options = parser.parse_args(args) + build_tree.chdir_to_root() + generator = TestGenerator(options) + if options.list: + for name in sorted(generator.TARGETS): + print(generator.filename_for(name)) + return + # List in a cmake list format (i.e. ';'-separated) + if options.list_for_cmake: + print(';'.join(generator.filename_for(name) + for name in sorted(generator.TARGETS)), end='') + return + if options.targets: + # Allow "-" as a special case so you can run + # ``generate_bignum_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] + else: + options.targets = sorted(generator.TARGETS) + for target in options.targets: + generator.generate_target(target) + +if __name__ == '__main__': + main(sys.argv[1:]) From 423f99bcef6c55b32af6f3389753e00c82322afe Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Mon, 18 Jul 2022 15:49:43 +0100 Subject: [PATCH 02/42] Add test generation for bignum cmp variant Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index c6e6a116caf7..36d0d229040e 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -194,6 +194,16 @@ def result(self): return str(self._result) +class BignumCmpAbs(BignumCmp): + """Target for abs comparison variant.""" + count = 0 + func = "mbedtls_mpi_cmp_abs" + title = "MPI compare (abs)" + + def __init__(self, val_l, val_r): + super().__init__(val_l.strip("-"), val_r.strip("-")) + + class TestGenerator: """Generate test data.""" From 5c1173bc1b97b9da3b1d19e609c9822f3ac8471e Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Mon, 18 Jul 2022 17:22:58 +0100 Subject: [PATCH 03/42] Add test case generation for bignum add Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 36d0d229040e..e8db99d091a0 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -204,6 +204,27 @@ def __init__(self, val_l, val_r): super().__init__(val_l.strip("-"), val_r.strip("-")) +class BignumAdd(BignumOperation): + """Target for bignum addition test cases.""" + count = 0 + func = "mbedtls_mpi_add_mpi" + title = "MPI add" + input_cases = list(itertools.combinations( + [ + "1c67967269c6", "9cde3", + "-1c67967269c6", "-9cde3", + ], 2 + )) + + def __init__(self, val_l, val_r): + super().__init__(val_l, val_r) + self.symb = "+" + + @property + def result(self): + return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) + + class TestGenerator: """Generate test data.""" From 1c413bda51aa0fc0af1c37c2b508c94e1dea2707 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 20 Jul 2022 13:35:22 +0100 Subject: [PATCH 04/42] Sort tests when generating cases Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index e8db99d091a0..76bce5e7bd0a 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -82,7 +82,7 @@ def create_test_case(self) -> test_case.TestCase: @classmethod def generate_tests(cls): """Generate test cases for the target subclasses.""" - for subclass in cls.__subclasses__(): + for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() From 1bdee226e3b123faa30ed07ec72919525cd6c4d5 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 20 Jul 2022 13:35:53 +0100 Subject: [PATCH 05/42] Remove set() to preserve test case order Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 76bce5e7bd0a..72a10616f77c 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -155,10 +155,9 @@ def val_desc(val) -> str: @classmethod def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: """Generate value pairs.""" - for pair in set( - list(itertools.combinations(cls.input_vals, 2)) + - cls.input_cases - ): + for pair in list( + itertools.combinations(cls.input_vals, 2) + ) + cls.input_cases: yield pair @classmethod From d76c5edc8ee87f167d0ff3b881044267be2e88bf Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 20 Jul 2022 14:13:44 +0100 Subject: [PATCH 06/42] Fix type issues Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 72a10616f77c..e15261500cb1 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -26,7 +26,7 @@ import posixpath import re import sys -from typing import Iterable, Iterator, Optional, Tuple, TypeVar +from typing import Iterable, Iterator, List, Optional, Tuple, TypeVar import scripts_path # pylint: disable=unused-import from mbedtls_dev import build_tree @@ -52,16 +52,16 @@ class BaseTarget: title: Description of the test function/purpose. """ count = 0 - desc = None - func = None + desc = "" + func = "" gen_file = "" - title = None + title = "" def __init__(self) -> None: type(self).count += 1 @property - def args(self) -> Iterable[str]: + def args(self) -> List[str]: """Create list of arguments for test case.""" return [] @@ -105,8 +105,8 @@ class BignumOperation(BignumTarget): "", "0", "7b", "-7b", "0000000000000000123", "-0000000000000000123", "1230000000000000000", "-1230000000000000000" - ] - input_cases = [] + ] # type: List[str] + input_cases = [] # type: List[Tuple[str, ...]] def __init__(self, val_l: str, val_r: str) -> None: super().__init__() From f0910ae380ae157666e44a8cbbbdf8ea8d14f239 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 20 Jul 2022 14:45:23 +0100 Subject: [PATCH 07/42] Remove is None from if statement Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index e15261500cb1..299b619e8cad 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -162,7 +162,7 @@ def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: @classmethod def generate_tests(cls) -> Iterator[test_case.TestCase]: - if cls.func is not None: + if cls.func: # Generate tests for the current class for l_value, r_value in cls.get_value_pairs(): cur_op = cls(l_value, r_value) From 92bb1cd8cb47a16d970d9f6b3bbb2bcce06116a1 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 20 Jul 2022 15:16:50 +0100 Subject: [PATCH 08/42] Fix incorrect indentation Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 299b619e8cad..61f642b37b36 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -156,7 +156,7 @@ def val_desc(val) -> str: def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: """Generate value pairs.""" for pair in list( - itertools.combinations(cls.input_vals, 2) + itertools.combinations(cls.input_vals, 2) ) + cls.input_cases: yield pair From dcad1e93fef044748aa4f98a9c823ba5e515ea9e Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 11:30:03 +0100 Subject: [PATCH 09/42] Separate common test generation classes/functions Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 138 +++++++++++++++++++++++++ tests/scripts/generate_bignum_tests.py | 129 ++--------------------- tests/scripts/generate_psa_tests.py | 71 ++----------- 3 files changed, 157 insertions(+), 181 deletions(-) create mode 100644 scripts/mbedtls_dev/test_generation.py diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py new file mode 100644 index 000000000000..3abedd00f4c5 --- /dev/null +++ b/scripts/mbedtls_dev/test_generation.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Common test generation classes and main function. + +These are used both by generate_psa_tests.py and generate_bignum_tests.py. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import posixpath +import re +from typing import Callable, Dict, Iterable, List, Type, TypeVar + +from mbedtls_dev import test_case + +T = TypeVar('T') #pylint: disable=invalid-name + + +class BaseTarget: + """Base target for test case generation. + + Attributes: + count: Counter for test class. + desc: Short description of test case. + func: Function which the class generates tests for. + gen_file: File to write generated tests to. + title: Description of the test function/purpose. + """ + count = 0 + desc = "" + func = "" + gen_file = "" + title = "" + + def __init__(self) -> None: + type(self).count += 1 + + @property + def args(self) -> List[str]: + """Create list of arguments for test case.""" + return [] + + @property + def description(self) -> str: + """Create a numbered test description.""" + return "{} #{} {}".format(self.title, self.count, self.desc) + + def create_test_case(self) -> test_case.TestCase: + """Generate test case from the current object.""" + tc = test_case.TestCase() + tc.set_description(self.description) + tc.set_function(self.func) + tc.set_arguments(self.args) + + return tc + + @classmethod + def generate_tests(cls): + """Generate test cases for the target subclasses.""" + for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): + yield from subclass.generate_tests() + + +class TestGenerator: + """Generate test data.""" + def __init__(self, options) -> None: + self.test_suite_directory = self.get_option(options, 'directory', + 'tests/suites') + + @staticmethod + def get_option(options, name: str, default: T) -> T: + value = getattr(options, name, None) + return default if value is None else value + + def filename_for(self, basename: str) -> str: + """The location of the data file with the specified base name.""" + return posixpath.join(self.test_suite_directory, basename + '.data') + + def write_test_data_file(self, basename: str, + test_cases: Iterable[test_case.TestCase]) -> None: + """Write the test cases to a .data file. + + The output file is ``basename + '.data'`` in the test suite directory. + """ + filename = self.filename_for(basename) + test_case.write_data_file(filename, test_cases) + + # Note that targets whose names contain 'test_format' have their content + # validated by `abi_check.py`. + TARGETS = {} # type: Dict[str, Callable[..., test_case.TestCase]] + + def generate_target(self, name: str, *target_args) -> None: + """Generate cases and write to data file for a target. + + For target callables which require arguments, override this function + and pass these arguments using super() (see PSATestGenerator). + """ + test_cases = self.TARGETS[name](*target_args) + self.write_test_data_file(name, test_cases) + +def main(args, generator_class: Type[TestGenerator] = TestGenerator): + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--list', action='store_true', + help='List available targets and exit') + parser.add_argument('targets', nargs='*', metavar='TARGET', + help='Target file to generate (default: all; "-": none)') + options = parser.parse_args(args) + generator = generator_class(options) + if options.list: + for name in sorted(generator.TARGETS): + print(generator.filename_for(name)) + return + if options.targets: + # Allow "-" as a special case so you can run + # ``generate_xxx_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] + else: + options.targets = sorted(generator.TARGETS) + for target in options.targets: + generator.generate_target(target) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 61f642b37b36..f885167cfa71 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -20,17 +20,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse import itertools -import os -import posixpath -import re import sys -from typing import Iterable, Iterator, List, Optional, Tuple, TypeVar +from typing import Callable, Dict, Iterator, List, Optional, Tuple, TypeVar import scripts_path # pylint: disable=unused-import -from mbedtls_dev import build_tree from mbedtls_dev import test_case +from mbedtls_dev import test_generation T = TypeVar('T') #pylint: disable=invalid-name @@ -41,52 +37,7 @@ def quote_str(val): return "\"{}\"".format(val) -class BaseTarget: - """Base target for test case generation. - - Attributes: - count: Counter for test class. - desc: Short description of test case. - func: Function which the class generates tests for. - gen_file: File to write generated tests to. - title: Description of the test function/purpose. - """ - count = 0 - desc = "" - func = "" - gen_file = "" - title = "" - - def __init__(self) -> None: - type(self).count += 1 - - @property - def args(self) -> List[str]: - """Create list of arguments for test case.""" - return [] - - @property - def description(self) -> str: - """Create a numbered test description.""" - return "{} #{} {}".format(self.title, self.count, self.desc) - - def create_test_case(self) -> test_case.TestCase: - """Generate test case from the current object.""" - tc = test_case.TestCase() - tc.set_description(self.description) - tc.set_function(self.func) - tc.set_arguments(self.args) - - return tc - - @classmethod - def generate_tests(cls): - """Generate test cases for the target subclasses.""" - for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): - yield from subclass.generate_tests() - - -class BignumTarget(BaseTarget): +class BignumTarget(test_generation.BaseTarget): """Target for bignum (mpi) test case generation.""" gen_file = 'test_suite_mpi.generated' @@ -224,76 +175,12 @@ def result(self): return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) -class TestGenerator: - """Generate test data.""" - - def __init__(self, options) -> None: - self.test_suite_directory = self.get_option(options, 'directory', - 'tests/suites') - - @staticmethod - def get_option(options, name: str, default: T) -> T: - value = getattr(options, name, None) - return default if value is None else value - - def filename_for(self, basename: str) -> str: - """The location of the data file with the specified base name.""" - return posixpath.join(self.test_suite_directory, basename + '.data') - - def write_test_data_file(self, basename: str, - test_cases: Iterable[test_case.TestCase]) -> None: - """Write the test cases to a .data file. - - The output file is ``basename + '.data'`` in the test suite directory. - """ - filename = self.filename_for(basename) - test_case.write_data_file(filename, test_cases) - - # Note that targets whose names contain 'test_format' have their content - # validated by `abi_check.py`. +class BignumTestGenerator(test_generation.TestGenerator): + """Test generator subclass including bignum targets.""" TARGETS = { subclass.gen_file: subclass.generate_tests for subclass in - BaseTarget.__subclasses__() - } - - def generate_target(self, name: str) -> None: - test_cases = self.TARGETS[name]() - self.write_test_data_file(name, test_cases) - -def main(args): - """Command line entry point.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--list', action='store_true', - help='List available targets and exit') - parser.add_argument('--list-for-cmake', action='store_true', - help='Print \';\'-separated list of available targets and exit') - parser.add_argument('--directory', metavar='DIR', - help='Output directory (default: tests/suites)') - parser.add_argument('targets', nargs='*', metavar='TARGET', - help='Target file to generate (default: all; "-": none)') - options = parser.parse_args(args) - build_tree.chdir_to_root() - generator = TestGenerator(options) - if options.list: - for name in sorted(generator.TARGETS): - print(generator.filename_for(name)) - return - # List in a cmake list format (i.e. ';'-separated) - if options.list_for_cmake: - print(';'.join(generator.filename_for(name) - for name in sorted(generator.TARGETS)), end='') - return - if options.targets: - # Allow "-" as a special case so you can run - # ``generate_bignum_tests.py - $targets`` and it works uniformly whether - # ``$targets`` is empty or not. - options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] - else: - options.targets = sorted(generator.TARGETS) - for target in options.targets: - generator.generate_target(target) + test_generation.BaseTarget.__subclasses__() + } # type: Dict[str, Callable[[], test_case.TestCase]] if __name__ == '__main__': - main(sys.argv[1:]) + test_generation.main(sys.argv[1:], BignumTestGenerator) diff --git a/tests/scripts/generate_psa_tests.py b/tests/scripts/generate_psa_tests.py index 33c1fc3d1c99..ebec2ee9bf3c 100755 --- a/tests/scripts/generate_psa_tests.py +++ b/tests/scripts/generate_psa_tests.py @@ -20,20 +20,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse import enum -import os import re import sys -from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional, TypeVar +from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional import scripts_path # pylint: disable=unused-import from mbedtls_dev import crypto_knowledge from mbedtls_dev import macro_collector from mbedtls_dev import psa_storage from mbedtls_dev import test_case - -T = TypeVar('T') #pylint: disable=invalid-name +from mbedtls_dev import test_generation def psa_want_symbol(name: str) -> str: @@ -897,32 +894,8 @@ def generate_all_keys(self) -> Iterator[StorageTestData]: yield from super().generate_all_keys() yield from self.all_keys_for_implicit_usage() -class TestGenerator: - """Generate test data.""" - - def __init__(self, options) -> None: - self.test_suite_directory = self.get_option(options, 'directory', - 'tests/suites') - self.info = Information() - - @staticmethod - def get_option(options, name: str, default: T) -> T: - value = getattr(options, name, None) - return default if value is None else value - - def filename_for(self, basename: str) -> str: - """The location of the data file with the specified base name.""" - return os.path.join(self.test_suite_directory, basename + '.data') - - def write_test_data_file(self, basename: str, - test_cases: Iterable[test_case.TestCase]) -> None: - """Write the test cases to a .data file. - - The output file is ``basename + '.data'`` in the test suite directory. - """ - filename = self.filename_for(basename) - test_case.write_data_file(filename, test_cases) - +class PSATestGenerator(test_generation.TestGenerator): + """Test generator subclass including PSA targets and info.""" # Note that targets whose names contain 'test_format' have their content # validated by `abi_check.py`. TARGETS = { @@ -938,34 +911,12 @@ def write_test_data_file(self, basename: str, lambda info: StorageFormatV0(info).all_test_cases(), } #type: Dict[str, Callable[[Information], Iterable[test_case.TestCase]]] - def generate_target(self, name: str) -> None: - test_cases = self.TARGETS[name](self.info) - self.write_test_data_file(name, test_cases) - -def main(args): - """Command line entry point.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--list', action='store_true', - help='List available targets and exit') - parser.add_argument('targets', nargs='*', metavar='TARGET', - help='Target file to generate (default: all; "-": none)') - options = parser.parse_args(args) - generator = TestGenerator(options) - if options.list: - for name in sorted(generator.TARGETS): - print(generator.filename_for(name)) - return - if options.targets: - # Allow "-" as a special case so you can run - # ``generate_psa_tests.py - $targets`` and it works uniformly whether - # ``$targets`` is empty or not. - options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] - else: - options.targets = sorted(generator.TARGETS) - for target in options.targets: - generator.generate_target(target) + def __init__(self, options): + super().__init__(options) + self.info = Information() + + def generate_target(self, name: str, *target_args) -> None: + super().generate_target(name, self.info) if __name__ == '__main__': - main(sys.argv[1:]) + test_generation.main(sys.argv[1:], PSATestGenerator) From 70d3f3dcdcb6bb3486c903462d28b68e94e76d10 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 23 Aug 2022 14:21:53 +0100 Subject: [PATCH 10/42] Remove abbreviations and clarify attributes Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 34 +++++++------- tests/scripts/generate_bignum_tests.py | 61 ++++++++++++-------------- 2 files changed, 45 insertions(+), 50 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 3abedd00f4c5..f72d64db579f 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -34,37 +34,37 @@ class BaseTarget: """Base target for test case generation. Attributes: - count: Counter for test class. - desc: Short description of test case. - func: Function which the class generates tests for. - gen_file: File to write generated tests to. - title: Description of the test function/purpose. + count: Counter for test cases from this class. + case_description: Short description of the test case. This may be + automatically generated using the class, or manually set. + target_basename: Basename of file to write generated tests to. This + should be specified in a child class of BaseTarget. + test_function: Test function which the class generates cases for. + test_name: A common name or description of the test function. This can + be the function's name, or a short summary of its purpose. """ count = 0 - desc = "" - func = "" - gen_file = "" - title = "" + case_description = "" + target_basename = "" + test_function = "" + test_name = "" def __init__(self) -> None: type(self).count += 1 - @property - def args(self) -> List[str]: - """Create list of arguments for test case.""" + def arguments(self) -> List[str]: return [] - @property def description(self) -> str: """Create a numbered test description.""" - return "{} #{} {}".format(self.title, self.count, self.desc) + return "{} #{} {}".format(self.test_name, self.count, self.case_description) def create_test_case(self) -> test_case.TestCase: """Generate test case from the current object.""" tc = test_case.TestCase() - tc.set_description(self.description) - tc.set_function(self.func) - tc.set_arguments(self.args) + tc.set_description(self.description()) + tc.set_function(self.test_function) + tc.set_arguments(self.arguments()) return tc diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index f885167cfa71..fbccb8a9f59b 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -39,20 +39,20 @@ def quote_str(val): class BignumTarget(test_generation.BaseTarget): """Target for bignum (mpi) test case generation.""" - gen_file = 'test_suite_mpi.generated' + target_basename = 'test_suite_mpi.generated' class BignumOperation(BignumTarget): """Common features for test cases covering bignum operations. Attributes: - symb: Symbol used for operation in description. - input_vals: List of values used to generate test case args. - input_cases: List of tuples containing test case inputs. This + symbol: Symbol used for operation in description. + input_values: List of values to use as test case inputs. + input_cases: List of tuples containing pairs of test case inputs. This can be used to implement specific pairs of inputs. """ - symb = "" - input_vals = [ + symbol = "" + input_values = [ "", "0", "7b", "-7b", "0000000000000000123", "-0000000000000000123", "1230000000000000000", "-1230000000000000000" @@ -67,26 +67,23 @@ def __init__(self, val_l: str, val_r: str) -> None: self.int_l = hex_to_int(val_l) self.int_r = hex_to_int(val_r) - @property - def args(self): - return [quote_str(self.arg_l), quote_str(self.arg_r), self.result] + def arguments(self): + return [quote_str(self.arg_l), quote_str(self.arg_r), self.result()] - @property def description(self): - desc = self.desc if self.desc else "{} {} {}".format( - self.val_desc(self.arg_l), - self.symb, - self.val_desc(self.arg_r) - ) - return "{} #{} {}".format(self.title, self.count, desc) - - @property + if not self.case_description: + self.case_description = "{} {} {}".format( + self.value_description(self.arg_l), + self.symbol, + self.value_description(self.arg_r) + ) + return super().description() + def result(self) -> Optional[str]: return None @staticmethod - def val_desc(val) -> str: - """Generate description of the argument val.""" + def value_description(val) -> str: if val == "": return "0 (null)" if val == "0": @@ -107,13 +104,13 @@ def val_desc(val) -> str: def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: """Generate value pairs.""" for pair in list( - itertools.combinations(cls.input_vals, 2) + itertools.combinations(cls.input_values, 2) ) + cls.input_cases: yield pair @classmethod def generate_tests(cls) -> Iterator[test_case.TestCase]: - if cls.func: + if cls.test_function: # Generate tests for the current class for l_value, r_value in cls.get_value_pairs(): cur_op = cls(l_value, r_value) @@ -125,8 +122,8 @@ def generate_tests(cls) -> Iterator[test_case.TestCase]: class BignumCmp(BignumOperation): """Target for bignum comparison test cases.""" count = 0 - func = "mbedtls_mpi_cmp_mpi" - title = "MPI compare" + test_function = "mbedtls_mpi_cmp_mpi" + test_name = "MPI compare" input_cases = [ ("-2", "-3"), ("-2", "-2"), @@ -137,9 +134,8 @@ class BignumCmp(BignumOperation): def __init__(self, val_l, val_r): super().__init__(val_l, val_r) self._result = (self.int_l > self.int_r) - (self.int_l < self.int_r) - self.symb = ["<", "==", ">"][self._result + 1] + self.symbol = ["<", "==", ">"][self._result + 1] - @property def result(self): return str(self._result) @@ -147,8 +143,8 @@ def result(self): class BignumCmpAbs(BignumCmp): """Target for abs comparison variant.""" count = 0 - func = "mbedtls_mpi_cmp_abs" - title = "MPI compare (abs)" + test_function = "mbedtls_mpi_cmp_abs" + test_name = "MPI compare (abs)" def __init__(self, val_l, val_r): super().__init__(val_l.strip("-"), val_r.strip("-")) @@ -157,8 +153,8 @@ def __init__(self, val_l, val_r): class BignumAdd(BignumOperation): """Target for bignum addition test cases.""" count = 0 - func = "mbedtls_mpi_add_mpi" - title = "MPI add" + test_function = "mbedtls_mpi_add_mpi" + test_name = "MPI add" input_cases = list(itertools.combinations( [ "1c67967269c6", "9cde3", @@ -168,9 +164,8 @@ class BignumAdd(BignumOperation): def __init__(self, val_l, val_r): super().__init__(val_l, val_r) - self.symb = "+" + self.symbol = "+" - @property def result(self): return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) @@ -178,7 +173,7 @@ def result(self): class BignumTestGenerator(test_generation.TestGenerator): """Test generator subclass including bignum targets.""" TARGETS = { - subclass.gen_file: subclass.generate_tests for subclass in + subclass.target_basename: subclass.generate_tests for subclass in test_generation.BaseTarget.__subclasses__() } # type: Dict[str, Callable[[], test_case.TestCase]] From 02998c470a67d042698ea5fdbf22cea3755bcbb8 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 23 Aug 2022 16:07:19 +0100 Subject: [PATCH 11/42] Remove unneeded list concatenation Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index fbccb8a9f59b..757a80a44368 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -103,10 +103,8 @@ def value_description(val) -> str: @classmethod def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: """Generate value pairs.""" - for pair in list( - itertools.combinations(cls.input_values, 2) - ) + cls.input_cases: - yield pair + yield from itertools.combinations(cls.input_values, 2) + yield from cls.input_cases @classmethod def generate_tests(cls) -> Iterator[test_case.TestCase]: From 1c2a732203d13e8a102172b1b43b11ed487dc136 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 16:37:44 +0100 Subject: [PATCH 12/42] Convert bools to int before arithmetic Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 757a80a44368..471fd7724571 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -131,7 +131,7 @@ class BignumCmp(BignumOperation): def __init__(self, val_l, val_r): super().__init__(val_l, val_r) - self._result = (self.int_l > self.int_r) - (self.int_l < self.int_r) + self._result = int(self.int_l > self.int_r) - int(self.int_l < self.int_r) self.symbol = ["<", "==", ">"][self._result + 1] def result(self): From 008d90d42461c5176e2b7eb6aa7d33e7de429663 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 23 Aug 2022 16:07:37 +0100 Subject: [PATCH 13/42] Add details to docstrings Clarification is added to docstrings, mostly in abstract classes. Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 38 +++++++++++++-- tests/scripts/generate_bignum_tests.py | 65 +++++++++++++++++++++++--- 2 files changed, 93 insertions(+), 10 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index f72d64db579f..ea391c332e5a 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -23,6 +23,8 @@ import os import posixpath import re + +from abc import abstractmethod from typing import Callable, Dict, Iterable, List, Type, TypeVar from mbedtls_dev import test_case @@ -52,15 +54,34 @@ class BaseTarget: def __init__(self) -> None: type(self).count += 1 + @abstractmethod def arguments(self) -> List[str]: - return [] + """Get the list of arguments for the test case. + + Override this method to provide the list of arguments required for + generating the test_function. + + Returns: + List of arguments required for the test function. + """ + pass def description(self) -> str: - """Create a numbered test description.""" + """Create a test description. + + Creates a description of the test case, including a name for the test + function, and describing the specific test case. This should inform a + reader of the purpose of the case. The case description may be + generated in the class, or provided manually as needed. + + Returns: + Description for the test case. + """ return "{} #{} {}".format(self.test_name, self.count, self.case_description) + def create_test_case(self) -> test_case.TestCase: - """Generate test case from the current object.""" + """Generate TestCase from the current object.""" tc = test_case.TestCase() tc.set_description(self.description()) tc.set_function(self.test_function) @@ -70,7 +91,16 @@ def create_test_case(self) -> test_case.TestCase: @classmethod def generate_tests(cls): - """Generate test cases for the target subclasses.""" + """Generate test cases for the target subclasses. + + Classes will iterate over its subclasses, calling this method in each. + In abstract classes, no further changes are needed, as there is no + function to generate tests for. + In classes which do implement a test function, this should be overrided + and a means to use `create_test_case()` should be added. In most cases + the subclasses can still be iterated over, as either the class will + have none, or it may continue. + """ for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 471fd7724571..7a8ebd1d8a51 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -3,6 +3,31 @@ With no arguments, generate all test data. With non-option arguments, generate only the specified files. + +Class structure: + +Target classes are directly derived from test_generation.BaseTarget, +representing a target file. These indicate where test cases will be written +to in classes derived from the Target. Multiple Target classes must not +represent the same target_basename. + +Each subclass derived from a Target can either be: + - A concrete class, representing a test function, which generates test cases. + - An abstract class containing shared methods and attributes, not associated + with a test function. An example is BignumOperation, which provides common + features used in binary bignum operations. + + +Adding test generation for a function: + +A subclass representing the test function should be added, deriving from a +Target class or a descendant. This subclass must set/implement the following: + - test_function: the function name from the associated .function file. + - arguments(): generation of the arguments required for the test_function. + - generate_function_test(): generation of the test cases for the function. + +Additional details and other attributes/methods are given in the documentation +of BaseTarget in test_generation.py. """ # Copyright The Mbed TLS Contributors @@ -22,6 +47,8 @@ import itertools import sys + +from abc import abstractmethod from typing import Callable, Dict, Iterator, List, Optional, Tuple, TypeVar import scripts_path # pylint: disable=unused-import @@ -43,11 +70,16 @@ class BignumTarget(test_generation.BaseTarget): class BignumOperation(BignumTarget): - """Common features for test cases covering bignum operations. + """Common features for test cases covering binary bignum operations. + + This adds functionality common in binary operation tests. This includes + generation of case descriptions, using descriptions of values and symbols + to represent the operation or result. Attributes: - symbol: Symbol used for operation in description. - input_values: List of values to use as test case inputs. + symbol: Symbol used for the operation in case description. + input_values: List of values to use as test case inputs. These are + combined to produce pairs of values. input_cases: List of tuples containing pairs of test case inputs. This can be used to implement specific pairs of inputs. """ @@ -71,6 +103,12 @@ def arguments(self): return [quote_str(self.arg_l), quote_str(self.arg_r), self.result()] def description(self): + """Generate a description for the test case. + + If not set, case_description uses the form A `symbol` B, where symbol + is used to represent the operation. Descriptions of each value are + generated to provide some context to the test case. + """ if not self.case_description: self.case_description = "{} {} {}".format( self.value_description(self.arg_l), @@ -79,11 +117,22 @@ def description(self): ) return super().description() + @abstractmethod def result(self) -> Optional[str]: - return None + """Get the result of the operation. + + This may be calculated during initialization and stored as `_result`, + or calculated when the method is called. + """ + pass @staticmethod def value_description(val) -> str: + """Generate a description of the argument val. + + This produces a simple description of the value, which are used in test + case naming, to avoid most generated cases only being numbered. + """ if val == "": return "0 (null)" if val == "0": @@ -102,7 +151,11 @@ def value_description(val) -> str: @classmethod def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: - """Generate value pairs.""" + """Generator for pairs of inputs. + + Combinations are first generated from all input values, and then + specific cases provided. + """ yield from itertools.combinations(cls.input_values, 2) yield from cls.input_cases @@ -139,7 +192,7 @@ def result(self): class BignumCmpAbs(BignumCmp): - """Target for abs comparison variant.""" + """Target for bignum comparison, absolute variant.""" count = 0 test_function = "mbedtls_mpi_cmp_abs" test_name = "MPI compare (abs)" From 47e37b3b75ab18386f2e1e3eea9c52a733a7cb6d Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 12:18:25 +0100 Subject: [PATCH 14/42] Use ABCMeta for abstract classes Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 15 +++++++-------- tests/scripts/generate_bignum_tests.py | 10 +++++----- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index ea391c332e5a..896f86ec1db4 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -24,7 +24,7 @@ import posixpath import re -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from typing import Callable, Dict, Iterable, List, Type, TypeVar from mbedtls_dev import test_case @@ -32,7 +32,7 @@ T = TypeVar('T') #pylint: disable=invalid-name -class BaseTarget: +class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. Attributes: @@ -93,13 +93,12 @@ def create_test_case(self) -> test_case.TestCase: def generate_tests(cls): """Generate test cases for the target subclasses. - Classes will iterate over its subclasses, calling this method in each. - In abstract classes, no further changes are needed, as there is no + During generation, each class will iterate over any subclasses, calling + this method in each. + In abstract classes, no tests will be generated, as there is no function to generate tests for. - In classes which do implement a test function, this should be overrided - and a means to use `create_test_case()` should be added. In most cases - the subclasses can still be iterated over, as either the class will - have none, or it may continue. + In classes which do implement a test function, this should be overridden + and a means to use `create_test_case()` should be added. """ for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 7a8ebd1d8a51..3f556ce29126 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -48,7 +48,7 @@ import itertools import sys -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from typing import Callable, Dict, Iterator, List, Optional, Tuple, TypeVar import scripts_path # pylint: disable=unused-import @@ -64,12 +64,12 @@ def quote_str(val): return "\"{}\"".format(val) -class BignumTarget(test_generation.BaseTarget): +class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta): """Target for bignum (mpi) test case generation.""" target_basename = 'test_suite_mpi.generated' -class BignumOperation(BignumTarget): +class BignumOperation(BignumTarget, metaclass=ABCMeta): """Common features for test cases covering binary bignum operations. This adds functionality common in binary operation tests. This includes @@ -118,7 +118,7 @@ def description(self): return super().description() @abstractmethod - def result(self) -> Optional[str]: + def result(self) -> str: """Get the result of the operation. This may be calculated during initialization and stored as `_result`, @@ -131,7 +131,7 @@ def value_description(val) -> str: """Generate a description of the argument val. This produces a simple description of the value, which are used in test - case naming, to avoid most generated cases only being numbered. + case naming, to add context to the test cases. """ if val == "": return "0 (null)" From c34d037fa0c272e2b94230026c9e9f560caca365 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 12:42:00 +0100 Subject: [PATCH 15/42] Split generate_tests to reduce code complexity Previous implementation mixed the test case generation and the recursive generation calls together. A separate method is added to generate test cases for the current class' test function. This reduces the need to override generate_tests(). Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 35 ++++++++++++++++++-------- tests/scripts/generate_bignum_tests.py | 12 +++------ 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 896f86ec1db4..59b3b08e4610 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -25,7 +25,7 @@ import re from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterable, List, Type, TypeVar +from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar from mbedtls_dev import test_case @@ -90,16 +90,31 @@ def create_test_case(self) -> test_case.TestCase: return tc @classmethod - def generate_tests(cls): - """Generate test cases for the target subclasses. - - During generation, each class will iterate over any subclasses, calling - this method in each. - In abstract classes, no tests will be generated, as there is no - function to generate tests for. - In classes which do implement a test function, this should be overridden - and a means to use `create_test_case()` should be added. + @abstractmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + """Generate test cases for the test function. + + This will be called in classes where `test_function` is set. + Implementations should yield TestCase objects, by creating instances + of the class with appropriate input data, and then calling + `create_test_case()` on each. + """ + pass + + @classmethod + def generate_tests(cls) -> Iterator[test_case.TestCase]: + """Generate test cases for the class and its subclasses. + + In classes with `test_function` set, `generate_function_tests()` is + used to generate test cases first. + In all classes, this method will iterate over its subclasses, and + yield from `generate_tests()` in each. + + Calling this method on a class X will yield test cases from all classes + derived from X. """ + if cls.test_function: + yield from cls.generate_function_tests() for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 3f556ce29126..1f644852895e 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -160,14 +160,10 @@ def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: yield from cls.input_cases @classmethod - def generate_tests(cls) -> Iterator[test_case.TestCase]: - if cls.test_function: - # Generate tests for the current class - for l_value, r_value in cls.get_value_pairs(): - cur_op = cls(l_value, r_value) - yield cur_op.create_test_case() - # Once current class completed, check descendants - yield from super().generate_tests() + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for l_value, r_value in cls.get_value_pairs(): + cur_op = cls(l_value, r_value) + yield cur_op.create_test_case() class BignumCmp(BignumOperation): From cace1aa02e92461d41a7e66605bf6b7bb6ec8d86 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 17:04:07 +0100 Subject: [PATCH 16/42] Use __new__() for case counting Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 5 +++-- tests/scripts/generate_bignum_tests.py | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 59b3b08e4610..9f4e296df928 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -51,8 +51,9 @@ class BaseTarget(metaclass=ABCMeta): test_function = "" test_name = "" - def __init__(self) -> None: - type(self).count += 1 + def __new__(cls, *args, **kwargs): + cls.count += 1 + return super().__new__(cls) @abstractmethod def arguments(self) -> List[str]: diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 1f644852895e..9551e2186749 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -92,8 +92,6 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): input_cases = [] # type: List[Tuple[str, ...]] def __init__(self, val_l: str, val_r: str) -> None: - super().__init__() - self.arg_l = val_l self.arg_r = val_r self.int_l = hex_to_int(val_l) From 6d04142972989e64b7172be62b54b7e9fb6fc3b6 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 17:20:29 +0100 Subject: [PATCH 17/42] Remove trailing whitespace in description Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 9f4e296df928..d2e2afdd44fa 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -78,7 +78,9 @@ def description(self) -> str: Returns: Description for the test case. """ - return "{} #{} {}".format(self.test_name, self.count, self.case_description) + return "{} #{} {}".format( + self.test_name, self.count, self.case_description + ).strip() def create_test_case(self) -> test_case.TestCase: From 9509f44d79b8b18c43922b72ed5e139919120050 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 17:46:22 +0100 Subject: [PATCH 18/42] Add missing typing Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 9551e2186749..016e03771202 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -49,7 +49,7 @@ import sys from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterator, List, Optional, Tuple, TypeVar +from typing import Callable, Dict, Iterator, List, Tuple, TypeVar import scripts_path # pylint: disable=unused-import from mbedtls_dev import test_case @@ -57,10 +57,10 @@ T = TypeVar('T') #pylint: disable=invalid-name -def hex_to_int(val): +def hex_to_int(val: str) -> int: return int(val, 16) if val else 0 -def quote_str(val): +def quote_str(val) -> str: return "\"{}\"".format(val) @@ -89,7 +89,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): "0000000000000000123", "-0000000000000000123", "1230000000000000000", "-1230000000000000000" ] # type: List[str] - input_cases = [] # type: List[Tuple[str, ...]] + input_cases = [] # type: List[Tuple[str, str]] def __init__(self, val_l: str, val_r: str) -> None: self.arg_l = val_l @@ -97,10 +97,10 @@ def __init__(self, val_l: str, val_r: str) -> None: self.int_l = hex_to_int(val_l) self.int_r = hex_to_int(val_r) - def arguments(self): + def arguments(self) -> List[str]: return [quote_str(self.arg_l), quote_str(self.arg_r), self.result()] - def description(self): + def description(self) -> str: """Generate a description for the test case. If not set, case_description uses the form A `symbol` B, where symbol @@ -148,7 +148,7 @@ def value_description(val) -> str: return tmp @classmethod - def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: + def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: """Generator for pairs of inputs. Combinations are first generated from all input values, and then @@ -176,12 +176,12 @@ class BignumCmp(BignumOperation): ("2b5", "2b6") ] - def __init__(self, val_l, val_r): + def __init__(self, val_l, val_r) -> None: super().__init__(val_l, val_r) self._result = int(self.int_l > self.int_r) - int(self.int_l < self.int_r) self.symbol = ["<", "==", ">"][self._result + 1] - def result(self): + def result(self) -> str: return str(self._result) @@ -191,7 +191,7 @@ class BignumCmpAbs(BignumCmp): test_function = "mbedtls_mpi_cmp_abs" test_name = "MPI compare (abs)" - def __init__(self, val_l, val_r): + def __init__(self, val_l, val_r) -> None: super().__init__(val_l.strip("-"), val_r.strip("-")) @@ -207,11 +207,11 @@ class BignumAdd(BignumOperation): ], 2 )) - def __init__(self, val_l, val_r): + def __init__(self, val_l, val_r) -> None: super().__init__(val_l, val_r) self.symbol = "+" - def result(self): + def result(self) -> str: return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) From 478a4ce1fedd7b2e3fcca53640f6d4d0d16b461c Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 18:03:30 +0100 Subject: [PATCH 19/42] Use typing casts for fixed-width tuples Enforces fixed-width tuple types where mypy does not recognize. Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 016e03771202..a2a9d0674cf2 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -49,7 +49,7 @@ import sys from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterator, List, Tuple, TypeVar +from typing import Callable, Dict, Iterator, List, Tuple, TypeVar, cast import scripts_path # pylint: disable=unused-import from mbedtls_dev import test_case @@ -89,7 +89,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): "0000000000000000123", "-0000000000000000123", "1230000000000000000", "-1230000000000000000" ] # type: List[str] - input_cases = [] # type: List[Tuple[str, str]] + input_cases = cast(List[Tuple[str, str]], []) # type: List[Tuple[str, str]] def __init__(self, val_l: str, val_r: str) -> None: self.arg_l = val_l @@ -154,7 +154,10 @@ def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: Combinations are first generated from all input values, and then specific cases provided. """ - yield from itertools.combinations(cls.input_values, 2) + yield from cast( + Iterator[Tuple[str, str]], + itertools.combinations(cls.input_values, 2) + ) yield from cls.input_cases @classmethod @@ -200,12 +203,15 @@ class BignumAdd(BignumOperation): count = 0 test_function = "mbedtls_mpi_add_mpi" test_name = "MPI add" - input_cases = list(itertools.combinations( + input_cases = cast( + List[Tuple[str, str]], + list(itertools.combinations( [ "1c67967269c6", "9cde3", "-1c67967269c6", "-9cde3", ], 2 - )) + )) + ) def __init__(self, val_l, val_r) -> None: super().__init__(val_l, val_r) From 486d25850f4bc6a57da5ae2f4e0636a812d77d38 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 18:09:10 +0100 Subject: [PATCH 20/42] Disable pylint unused arg in __new__ Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index d2e2afdd44fa..582b54264b71 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -52,6 +52,7 @@ class BaseTarget(metaclass=ABCMeta): test_name = "" def __new__(cls, *args, **kwargs): + # pylint: disable=unused-argument cls.count += 1 return super().__new__(cls) From d77d33defb238044b94ced7affa14c3e144e46fa Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 09:56:51 +0100 Subject: [PATCH 21/42] Raise NotImplementedError in abstract methods Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 4 ++-- tests/scripts/generate_bignum_tests.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 582b54264b71..55076dc2939d 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -66,7 +66,7 @@ def arguments(self) -> List[str]: Returns: List of arguments required for the test function. """ - pass + raise NotImplementedError def description(self) -> str: """Create a test description. @@ -103,7 +103,7 @@ def generate_function_tests(cls) -> Iterator[test_case.TestCase]: of the class with appropriate input data, and then calling `create_test_case()` on each. """ - pass + raise NotImplementedError @classmethod def generate_tests(cls) -> Iterator[test_case.TestCase]: diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index a2a9d0674cf2..aa7e131a1d82 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -122,7 +122,7 @@ def result(self) -> str: This may be calculated during initialization and stored as `_result`, or calculated when the method is called. """ - pass + raise NotImplementedError @staticmethod def value_description(val) -> str: From 412c497cbef87b08bb970723deb0cf3ec71a154a Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 10:02:06 +0100 Subject: [PATCH 22/42] Fix TARGET types and code style Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 3 +-- tests/scripts/generate_bignum_tests.py | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 55076dc2939d..64b8fe41475b 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Common test generation classes and main function. These are used both by generate_psa_tests.py and generate_bignum_tests.py. @@ -149,7 +148,7 @@ def write_test_data_file(self, basename: str, # Note that targets whose names contain 'test_format' have their content # validated by `abi_check.py`. - TARGETS = {} # type: Dict[str, Callable[..., test_case.TestCase]] + TARGETS = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] def generate_target(self, name: str, *target_args) -> None: """Generate cases and write to data file for a target. diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index aa7e131a1d82..c57f197a2128 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -49,7 +49,7 @@ import sys from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterator, List, Tuple, TypeVar, cast +from typing import Callable, Dict, Iterable, Iterator, List, Tuple, TypeVar, cast import scripts_path # pylint: disable=unused-import from mbedtls_dev import test_case @@ -206,10 +206,10 @@ class BignumAdd(BignumOperation): input_cases = cast( List[Tuple[str, str]], list(itertools.combinations( - [ - "1c67967269c6", "9cde3", - "-1c67967269c6", "-9cde3", - ], 2 + [ + "1c67967269c6", "9cde3", + "-1c67967269c6", "-9cde3", + ], 2 )) ) @@ -226,7 +226,7 @@ class BignumTestGenerator(test_generation.TestGenerator): TARGETS = { subclass.target_basename: subclass.generate_tests for subclass in test_generation.BaseTarget.__subclasses__() - } # type: Dict[str, Callable[[], test_case.TestCase]] + } # type: Dict[str, Callable[[], Iterable[test_case.TestCase]]] if __name__ == '__main__': test_generation.main(sys.argv[1:], BignumTestGenerator) From b29f59f5ff949421a4c4577947606d5b47e1fbb5 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 11:17:35 +0100 Subject: [PATCH 23/42] Disable abstract check in pylint Version of pylint used in CI does not recognize abstract subclasses of BaseTarget, so disable warning in these abstract classes. Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index c57f197a2128..2443f659b308 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -65,6 +65,7 @@ def quote_str(val) -> str: class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta): + #pylint: disable=abstract-method """Target for bignum (mpi) test case generation.""" target_basename = 'test_suite_mpi.generated' From b03420fb3b8d9bcea619f1912ddb5a0d7a0bdee8 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 12:29:46 +0100 Subject: [PATCH 24/42] Clarify documentation Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 26 ++++++++------- tests/scripts/generate_bignum_tests.py | 44 +++++++++++++++----------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 64b8fe41475b..997d8e1d35be 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -34,6 +34,8 @@ class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. + This should be derived from for file Targets. + Attributes: count: Counter for test cases from this class. case_description: Short description of the test case. This may be @@ -42,7 +44,8 @@ class BaseTarget(metaclass=ABCMeta): should be specified in a child class of BaseTarget. test_function: Test function which the class generates cases for. test_name: A common name or description of the test function. This can - be the function's name, or a short summary of its purpose. + be `test_function`, a clearer equivalent, or a short summary of the + test function's purpose. """ count = 0 case_description = "" @@ -60,7 +63,7 @@ def arguments(self) -> List[str]: """Get the list of arguments for the test case. Override this method to provide the list of arguments required for - generating the test_function. + the `test_function`. Returns: List of arguments required for the test function. @@ -68,12 +71,12 @@ def arguments(self) -> List[str]: raise NotImplementedError def description(self) -> str: - """Create a test description. + """Create a test case description. Creates a description of the test case, including a name for the test - function, and describing the specific test case. This should inform a - reader of the purpose of the case. The case description may be - generated in the class, or provided manually as needed. + function, a case number, and a description the specific test case. + This should inform a reader what is being tested, and provide context + for the test case. Returns: Description for the test case. @@ -84,7 +87,7 @@ def description(self) -> str: def create_test_case(self) -> test_case.TestCase: - """Generate TestCase from the current object.""" + """Generate TestCase from the instance.""" tc = test_case.TestCase() tc.set_description(self.description()) tc.set_function(self.test_function) @@ -95,7 +98,7 @@ def create_test_case(self) -> test_case.TestCase: @classmethod @abstractmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - """Generate test cases for the test function. + """Generate test cases for the class test function. This will be called in classes where `test_function` is set. Implementations should yield TestCase objects, by creating instances @@ -110,11 +113,10 @@ def generate_tests(cls) -> Iterator[test_case.TestCase]: In classes with `test_function` set, `generate_function_tests()` is used to generate test cases first. - In all classes, this method will iterate over its subclasses, and - yield from `generate_tests()` in each. - Calling this method on a class X will yield test cases from all classes - derived from X. + In all classes, this method will iterate over its subclasses, and + yield from `generate_tests()` in each. Calling this method on a class X + will yield test cases from all classes derived from X. """ if cls.test_function: yield from cls.generate_function_tests() diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 2443f659b308..4486d49584ee 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -6,25 +6,33 @@ Class structure: -Target classes are directly derived from test_generation.BaseTarget, -representing a target file. These indicate where test cases will be written -to in classes derived from the Target. Multiple Target classes must not -represent the same target_basename. +Child classes of test_generation.BaseTarget (file Targets) represent a target +file. These indicate where test cases will be written to, for all subclasses of +this Target. Multiple Target classes should not reuse a `target_basename`. -Each subclass derived from a Target can either be: +Each subclass derived from a file Target can either be: - A concrete class, representing a test function, which generates test cases. - An abstract class containing shared methods and attributes, not associated - with a test function. An example is BignumOperation, which provides common - features used in binary bignum operations. + with a test function. An example is BignumOperation, which provides + common features used for bignum binary operations. +Both concrete and abstract subclasses can be derived from, to implement +additional test cases (see BignumCmp and BignumCmpAbs for examples of deriving +from abstract and concrete classes). -Adding test generation for a function: + +Adding test case generation for a function: A subclass representing the test function should be added, deriving from a -Target class or a descendant. This subclass must set/implement the following: +file Target. This test class must set/implement the following: - test_function: the function name from the associated .function file. - - arguments(): generation of the arguments required for the test_function. - - generate_function_test(): generation of the test cases for the function. + - test_name: a descriptive name or brief summary to refer to the test + function. + - arguments(): a method to generate the list of arguments required for the + test_function. + - generate_function_test(): a method to generate TestCases for the function. + This should create instances of the class with required input data, and + call `.create_test_case()` to yield the TestCase. Additional details and other attributes/methods are given in the documentation of BaseTarget in test_generation.py. @@ -71,7 +79,7 @@ class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta): class BignumOperation(BignumTarget, metaclass=ABCMeta): - """Common features for test cases covering binary bignum operations. + """Common features for bignum binary operations. This adds functionality common in binary operation tests. This includes generation of case descriptions, using descriptions of values and symbols @@ -130,7 +138,7 @@ def value_description(val) -> str: """Generate a description of the argument val. This produces a simple description of the value, which are used in test - case naming, to add context to the test cases. + case naming, to add context. """ if val == "": return "0 (null)" @@ -150,7 +158,7 @@ def value_description(val) -> str: @classmethod def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: - """Generator for pairs of inputs. + """Generator to yield pairs of inputs. Combinations are first generated from all input values, and then specific cases provided. @@ -169,7 +177,7 @@ def generate_function_tests(cls) -> Iterator[test_case.TestCase]: class BignumCmp(BignumOperation): - """Target for bignum comparison test cases.""" + """Test cases for bignum value comparison.""" count = 0 test_function = "mbedtls_mpi_cmp_mpi" test_name = "MPI compare" @@ -190,7 +198,7 @@ def result(self) -> str: class BignumCmpAbs(BignumCmp): - """Target for bignum comparison, absolute variant.""" + """Test cases for absolute bignum value comparison.""" count = 0 test_function = "mbedtls_mpi_cmp_abs" test_name = "MPI compare (abs)" @@ -200,7 +208,7 @@ def __init__(self, val_l, val_r) -> None: class BignumAdd(BignumOperation): - """Target for bignum addition test cases.""" + """Test cases for bignum value addition.""" count = 0 test_function = "mbedtls_mpi_add_mpi" test_name = "MPI add" @@ -223,7 +231,7 @@ def result(self) -> str: class BignumTestGenerator(test_generation.TestGenerator): - """Test generator subclass including bignum targets.""" + """Test generator subclass setting bignum targets.""" TARGETS = { subclass.target_basename: subclass.generate_tests for subclass in test_generation.BaseTarget.__subclasses__() From ac86390da8b539fee20b4bc1f39e2c6ff0dc7d1a Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 12:49:41 +0100 Subject: [PATCH 25/42] Use argparser default for targets Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 997d8e1d35be..afe00409f363 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -167,6 +167,7 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): parser.add_argument('--list', action='store_true', help='List available targets and exit') parser.add_argument('targets', nargs='*', metavar='TARGET', + default=sorted(generator_class.TARGETS), help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) generator = generator_class(options) @@ -174,14 +175,11 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): for name in sorted(generator.TARGETS): print(generator.filename_for(name)) return - if options.targets: - # Allow "-" as a special case so you can run - # ``generate_xxx_tests.py - $targets`` and it works uniformly whether - # ``$targets`` is empty or not. - options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] - else: - options.targets = sorted(generator.TARGETS) + # Allow "-" as a special case so you can run + # ``generate_xxx_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] for target in options.targets: generator.generate_target(target) From 6f67bae527cf4f501d8ff724a82050c4eecde3e1 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 13:21:45 +0100 Subject: [PATCH 26/42] Fix trailing whitespace Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index afe00409f363..3ad699c91752 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -167,7 +167,7 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): parser.add_argument('--list', action='store_true', help='List available targets and exit') parser.add_argument('targets', nargs='*', metavar='TARGET', - default=sorted(generator_class.TARGETS), + default=sorted(generator_class.TARGETS), help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) generator = generator_class(options) From 2b0f7d8d56ae900fcb7fb66f40cc9df37a4af173 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 16:27:05 +0100 Subject: [PATCH 27/42] Modify wording in docstrings Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 5 +++-- tests/scripts/generate_bignum_tests.py | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 3ad699c91752..727f8d5f6b26 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -34,7 +34,8 @@ class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. - This should be derived from for file Targets. + Derive directly from this class when adding new file Targets, setting + `target_basename`. Attributes: count: Counter for test cases from this class. @@ -112,7 +113,7 @@ def generate_tests(cls) -> Iterator[test_case.TestCase]: """Generate test cases for the class and its subclasses. In classes with `test_function` set, `generate_function_tests()` is - used to generate test cases first. + called to generate test cases first. In all classes, this method will iterate over its subclasses, and yield from `generate_tests()` in each. Calling this method on a class X diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 4486d49584ee..8a8425e1cad3 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -24,7 +24,8 @@ Adding test case generation for a function: A subclass representing the test function should be added, deriving from a -file Target. This test class must set/implement the following: +file Target such as BignumTarget. This test class must set/implement the +following: - test_function: the function name from the associated .function file. - test_name: a descriptive name or brief summary to refer to the test function. @@ -128,8 +129,8 @@ def description(self) -> str: def result(self) -> str: """Get the result of the operation. - This may be calculated during initialization and stored as `_result`, - or calculated when the method is called. + This could be calculated during initialization and stored as `_result` + and then returned, or calculated when the method is called. """ raise NotImplementedError @@ -137,8 +138,8 @@ def result(self) -> str: def value_description(val) -> str: """Generate a description of the argument val. - This produces a simple description of the value, which are used in test - case naming, to add context. + This produces a simple description of the value, which is used in test + case naming to add context. """ if val == "": return "0 (null)" @@ -231,7 +232,7 @@ def result(self) -> str: class BignumTestGenerator(test_generation.TestGenerator): - """Test generator subclass setting bignum targets.""" + """Test generator subclass, for bignum file Targets.""" TARGETS = { subclass.target_basename: subclass.generate_tests for subclass in test_generation.BaseTarget.__subclasses__() From 18f94d8175936781ec2f3d7aa6469f2392404ef3 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 31 Aug 2022 16:55:44 +0100 Subject: [PATCH 28/42] Use `combinations_with_replacement` for inputs When generating combinations of values, `itertools.combinations` will not allow inputs to be repeated. This is replaced so that cases where input values match are generated, i.e. ("0", "0"). Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 8a8425e1cad3..b08ba378523b 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -166,7 +166,7 @@ def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: """ yield from cast( Iterator[Tuple[str, str]], - itertools.combinations(cls.input_values, 2) + itertools.combinations_with_replacement(cls.input_values, 2) ) yield from cls.input_cases @@ -215,7 +215,7 @@ class BignumAdd(BignumOperation): test_name = "MPI add" input_cases = cast( List[Tuple[str, str]], - list(itertools.combinations( + list(itertools.combinations_with_replacement( [ "1c67967269c6", "9cde3", "-1c67967269c6", "-9cde3", From 486b3410a4f58c8981cca2c9d02a38a7f29154f8 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 31 Aug 2022 17:01:38 +0100 Subject: [PATCH 29/42] Add dependencies attribute to BaseTarget Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 727f8d5f6b26..00d559af6a1d 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -41,6 +41,7 @@ class BaseTarget(metaclass=ABCMeta): count: Counter for test cases from this class. case_description: Short description of the test case. This may be automatically generated using the class, or manually set. + dependencies: A list of dependencies required for the test case. target_basename: Basename of file to write generated tests to. This should be specified in a child class of BaseTarget. test_function: Test function which the class generates cases for. @@ -50,6 +51,7 @@ class BaseTarget(metaclass=ABCMeta): """ count = 0 case_description = "" + dependencies: List[str] = [] target_basename = "" test_function = "" test_name = "" @@ -93,6 +95,7 @@ def create_test_case(self) -> test_case.TestCase: tc.set_description(self.description()) tc.set_function(self.test_function) tc.set_arguments(self.arguments()) + tc.set_dependencies(self.dependencies) return tc From 6cc5e5f0d9c68d96e7fa0937f479378d5ad0eab2 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 31 Aug 2022 17:16:44 +0100 Subject: [PATCH 30/42] Use Python 3.5 style typing for dependencies Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 00d559af6a1d..6382a991330e 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -51,7 +51,7 @@ class BaseTarget(metaclass=ABCMeta): """ count = 0 case_description = "" - dependencies: List[str] = [] + dependencies = [] # type: List[str] target_basename = "" test_function = "" test_name = "" From 0d07e86a4451649e2da90428518f5d86f4870234 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 2 Sep 2022 11:56:34 +0100 Subject: [PATCH 31/42] Rework TestGenerator to add file targets BaseTarget-derived targets are now added to TestGenerator.targets in initialization. This reduces repeated code in generate_xxx_tests.py scripts which use this framework. Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 27 ++++++++++++++++---------- tests/scripts/generate_bignum_tests.py | 9 +-------- tests/scripts/generate_psa_tests.py | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 6382a991330e..5341c9a6d1d3 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -133,6 +133,11 @@ class TestGenerator: def __init__(self, options) -> None: self.test_suite_directory = self.get_option(options, 'directory', 'tests/suites') + # Add file Targets which have been declared in other modules + self.targets.update({ + subclass.target_basename: subclass.generate_tests + for subclass in BaseTarget.__subclasses__() + }) @staticmethod def get_option(options, name: str, default: T) -> T: @@ -154,7 +159,7 @@ def write_test_data_file(self, basename: str, # Note that targets whose names contain 'test_format' have their content # validated by `abi_check.py`. - TARGETS = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] + targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] def generate_target(self, name: str, *target_args) -> None: """Generate cases and write to data file for a target. @@ -162,7 +167,7 @@ def generate_target(self, name: str, *target_args) -> None: For target callables which require arguments, override this function and pass these arguments using super() (see PSATestGenerator). """ - test_cases = self.TARGETS[name](*target_args) + test_cases = self.targets[name](*target_args) self.write_test_data_file(name, test_cases) def main(args, generator_class: Type[TestGenerator] = TestGenerator): @@ -171,19 +176,21 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): parser.add_argument('--list', action='store_true', help='List available targets and exit') parser.add_argument('targets', nargs='*', metavar='TARGET', - default=sorted(generator_class.TARGETS), help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) generator = generator_class(options) if options.list: - for name in sorted(generator.TARGETS): + for name in sorted(generator.targets): print(generator.filename_for(name)) return - # Allow "-" as a special case so you can run - # ``generate_xxx_tests.py - $targets`` and it works uniformly whether - # ``$targets`` is empty or not. - options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] + if options.targets: + # Allow "-" as a special case so you can run + # ``generate_xxx_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] + else: + options.targets = sorted(generator.targets) for target in options.targets: generator.generate_target(target) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index b08ba378523b..f6136804bec8 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -231,12 +231,5 @@ def result(self) -> str: return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) -class BignumTestGenerator(test_generation.TestGenerator): - """Test generator subclass, for bignum file Targets.""" - TARGETS = { - subclass.target_basename: subclass.generate_tests for subclass in - test_generation.BaseTarget.__subclasses__() - } # type: Dict[str, Callable[[], Iterable[test_case.TestCase]]] - if __name__ == '__main__': - test_generation.main(sys.argv[1:], BignumTestGenerator) + test_generation.main(sys.argv[1:]) diff --git a/tests/scripts/generate_psa_tests.py b/tests/scripts/generate_psa_tests.py index ebec2ee9bf3c..1f516c9928fe 100755 --- a/tests/scripts/generate_psa_tests.py +++ b/tests/scripts/generate_psa_tests.py @@ -898,7 +898,7 @@ class PSATestGenerator(test_generation.TestGenerator): """Test generator subclass including PSA targets and info.""" # Note that targets whose names contain 'test_format' have their content # validated by `abi_check.py`. - TARGETS = { + targets = { 'test_suite_psa_crypto_generate_key.generated': lambda info: KeyGenerate(info).test_cases_for_key_generation(), 'test_suite_psa_crypto_not_supported.generated': From e53be35c09583cab1229a81d1ec77176b3e05067 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 2 Sep 2022 12:57:37 +0100 Subject: [PATCH 32/42] Remove unused imports Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 4 ++-- tests/scripts/generate_bignum_tests.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 5341c9a6d1d3..83c91cc87d6f 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -188,8 +188,8 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): # ``generate_xxx_tests.py - $targets`` and it works uniformly whether # ``$targets`` is empty or not. options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] + for target in options.targets + if target != '-'] else: options.targets = sorted(generator.targets) for target in options.targets: diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index f6136804bec8..2a8107725e09 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -58,7 +58,7 @@ import sys from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterable, Iterator, List, Tuple, TypeVar, cast +from typing import Iterator, List, Tuple, TypeVar, cast import scripts_path # pylint: disable=unused-import from mbedtls_dev import test_case From 3edcee72c41d8553ecff6a9257e47423da57337f Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 2 Sep 2022 17:26:19 +0100 Subject: [PATCH 33/42] Use simpler int to hex string conversion Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 2a8107725e09..cc4db4c59eb6 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -228,7 +228,7 @@ def __init__(self, val_l, val_r) -> None: self.symbol = "+" def result(self) -> str: - return quote_str(hex(self.int_l + self.int_r).replace("0x", "", 1)) + return quote_str("{:x}".format(self.int_l + self.int_r)) if __name__ == '__main__': From 46c09a66dea25d50aa4280315d84880ea4622632 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Mon, 12 Sep 2022 17:34:15 +0100 Subject: [PATCH 34/42] Move symbol definition out of __init__ Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index cc4db4c59eb6..28d29bfe176e 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -211,6 +211,7 @@ def __init__(self, val_l, val_r) -> None: class BignumAdd(BignumOperation): """Test cases for bignum value addition.""" count = 0 + symbol = "+" test_function = "mbedtls_mpi_add_mpi" test_name = "MPI add" input_cases = cast( @@ -223,10 +224,6 @@ class BignumAdd(BignumOperation): )) ) - def __init__(self, val_l, val_r) -> None: - super().__init__(val_l, val_r) - self.symbol = "+" - def result(self) -> str: return quote_str("{:x}".format(self.int_l + self.int_r)) From 8b2d14bbd755206576d7d15f12da2dd5ab2d1d6a Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Mon, 12 Sep 2022 17:35:27 +0100 Subject: [PATCH 35/42] Replace L/R inputs with A/B Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 28d29bfe176e..3f60a0915f1d 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -101,14 +101,14 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): ] # type: List[str] input_cases = cast(List[Tuple[str, str]], []) # type: List[Tuple[str, str]] - def __init__(self, val_l: str, val_r: str) -> None: - self.arg_l = val_l - self.arg_r = val_r - self.int_l = hex_to_int(val_l) - self.int_r = hex_to_int(val_r) + def __init__(self, val_a: str, val_b: str) -> None: + self.arg_a = val_a + self.arg_b = val_b + self.int_a = hex_to_int(val_a) + self.int_b = hex_to_int(val_b) def arguments(self) -> List[str]: - return [quote_str(self.arg_l), quote_str(self.arg_r), self.result()] + return [quote_str(self.arg_a), quote_str(self.arg_b), self.result()] def description(self) -> str: """Generate a description for the test case. @@ -119,9 +119,9 @@ def description(self) -> str: """ if not self.case_description: self.case_description = "{} {} {}".format( - self.value_description(self.arg_l), + self.value_description(self.arg_a), self.symbol, - self.value_description(self.arg_r) + self.value_description(self.arg_b) ) return super().description() @@ -172,8 +172,8 @@ def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - for l_value, r_value in cls.get_value_pairs(): - cur_op = cls(l_value, r_value) + for a_value, b_value in cls.get_value_pairs(): + cur_op = cls(a_value, b_value) yield cur_op.create_test_case() @@ -189,9 +189,9 @@ class BignumCmp(BignumOperation): ("2b5", "2b6") ] - def __init__(self, val_l, val_r) -> None: - super().__init__(val_l, val_r) - self._result = int(self.int_l > self.int_r) - int(self.int_l < self.int_r) + def __init__(self, val_a, val_b) -> None: + super().__init__(val_a, val_b) + self._result = int(self.int_a > self.int_b) - int(self.int_a < self.int_b) self.symbol = ["<", "==", ">"][self._result + 1] def result(self) -> str: @@ -204,8 +204,8 @@ class BignumCmpAbs(BignumCmp): test_function = "mbedtls_mpi_cmp_abs" test_name = "MPI compare (abs)" - def __init__(self, val_l, val_r) -> None: - super().__init__(val_l.strip("-"), val_r.strip("-")) + def __init__(self, val_a, val_b) -> None: + super().__init__(val_a.strip("-"), val_b.strip("-")) class BignumAdd(BignumOperation): @@ -225,7 +225,7 @@ class BignumAdd(BignumOperation): ) def result(self) -> str: - return quote_str("{:x}".format(self.int_l + self.int_r)) + return quote_str("{:x}".format(self.int_a + self.int_b)) if __name__ == '__main__': From f518276457c0c4bfc3a03495bfae63817337877f Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 12:59:32 +0100 Subject: [PATCH 36/42] Update comments/docstrings in TestGenerator Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index 83c91cc87d6f..bba2f9fa06a6 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -129,11 +129,13 @@ def generate_tests(cls) -> Iterator[test_case.TestCase]: class TestGenerator: - """Generate test data.""" + """Generate test cases and write to data files.""" def __init__(self, options) -> None: self.test_suite_directory = self.get_option(options, 'directory', 'tests/suites') - # Add file Targets which have been declared in other modules + # Update `targets` with an entry for each child class of BaseTarget. + # Each entry represents a file generated by the BaseTarget framework, + # and enables generating the .data files using the CLI. self.targets.update({ subclass.target_basename: subclass.generate_tests for subclass in BaseTarget.__subclasses__() From 113ddd0df69cf5a14609356ceb175f614bab0b83 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 13:02:40 +0100 Subject: [PATCH 37/42] Add toggle for test case count in descriptions Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index bba2f9fa06a6..eb5c7631f796 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -42,6 +42,7 @@ class BaseTarget(metaclass=ABCMeta): case_description: Short description of the test case. This may be automatically generated using the class, or manually set. dependencies: A list of dependencies required for the test case. + show_test_count: Toggle for inclusion of `count` in the test description. target_basename: Basename of file to write generated tests to. This should be specified in a child class of BaseTarget. test_function: Test function which the class generates cases for. @@ -52,6 +53,7 @@ class BaseTarget(metaclass=ABCMeta): count = 0 case_description = "" dependencies = [] # type: List[str] + show_test_count = True target_basename = "" test_function = "" test_name = "" @@ -77,16 +79,19 @@ def description(self) -> str: """Create a test case description. Creates a description of the test case, including a name for the test - function, a case number, and a description the specific test case. - This should inform a reader what is being tested, and provide context - for the test case. + function, an optional case count, and a description of the specific + test case. This should inform a reader what is being tested, and + provide context for the test case. Returns: Description for the test case. """ - return "{} #{} {}".format( - self.test_name, self.count, self.case_description - ).strip() + if self.show_test_count: + return "{} #{} {}".format( + self.test_name, self.count, self.case_description + ).strip() + else: + return "{} {}".format(self.test_name, self.case_description).strip() def create_test_case(self) -> test_case.TestCase: From 1965d48cad64bacf848afc2a37b72d360c81799a Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 15:00:22 +0100 Subject: [PATCH 38/42] Use typing.cast instead of unqualified cast Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 3f60a0915f1d..3453b6bc3289 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -56,9 +56,10 @@ import itertools import sys +import typing from abc import ABCMeta, abstractmethod -from typing import Iterator, List, Tuple, TypeVar, cast +from typing import Iterator, List, Tuple, TypeVar import scripts_path # pylint: disable=unused-import from mbedtls_dev import test_case @@ -99,7 +100,7 @@ class BignumOperation(BignumTarget, metaclass=ABCMeta): "0000000000000000123", "-0000000000000000123", "1230000000000000000", "-1230000000000000000" ] # type: List[str] - input_cases = cast(List[Tuple[str, str]], []) # type: List[Tuple[str, str]] + input_cases = [] # type: List[Tuple[str, str]] def __init__(self, val_a: str, val_b: str) -> None: self.arg_a = val_a @@ -164,7 +165,7 @@ def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: Combinations are first generated from all input values, and then specific cases provided. """ - yield from cast( + yield from typing.cast( Iterator[Tuple[str, str]], itertools.combinations_with_replacement(cls.input_values, 2) ) @@ -214,7 +215,7 @@ class BignumAdd(BignumOperation): symbol = "+" test_function = "mbedtls_mpi_add_mpi" test_name = "MPI add" - input_cases = cast( + input_cases = typing.cast( List[Tuple[str, str]], list(itertools.combinations_with_replacement( [ From 38c2491ef4a61acacc70d9f7db1b7069ed652a71 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 15:12:46 +0100 Subject: [PATCH 39/42] Add combination_pairs helper function Wrapper function for itertools.combinations_with_replacement, with explicit cast due to imprecise typing with older versions of mypy. Signed-off-by: Werner Lewis --- tests/scripts/generate_bignum_tests.py | 29 +++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 3453b6bc3289..d156f56f8f90 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -73,6 +73,17 @@ def hex_to_int(val: str) -> int: def quote_str(val) -> str: return "\"{}\"".format(val) +def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: + """Return all pair combinations from input values. + + The return value is cast, as older versions of mypy are unable to derive + the specific type returned by itertools.combinations_with_replacement. + """ + return typing.cast( + List[Tuple[T, T]], + list(itertools.combinations_with_replacement(values, 2)) + ) + class BignumTarget(test_generation.BaseTarget, metaclass=ABCMeta): #pylint: disable=abstract-method @@ -165,10 +176,7 @@ def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: Combinations are first generated from all input values, and then specific cases provided. """ - yield from typing.cast( - Iterator[Tuple[str, str]], - itertools.combinations_with_replacement(cls.input_values, 2) - ) + yield from combination_pairs(cls.input_values) yield from cls.input_cases @classmethod @@ -215,14 +223,11 @@ class BignumAdd(BignumOperation): symbol = "+" test_function = "mbedtls_mpi_add_mpi" test_name = "MPI add" - input_cases = typing.cast( - List[Tuple[str, str]], - list(itertools.combinations_with_replacement( - [ - "1c67967269c6", "9cde3", - "-1c67967269c6", "-9cde3", - ], 2 - )) + input_cases = combination_pairs( + [ + "1c67967269c6", "9cde3", + "-1c67967269c6", "-9cde3", + ] ) def result(self) -> str: From 64334d96d05946b2550e69b962b451d710f00b51 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 16:26:54 +0100 Subject: [PATCH 40/42] Update references to file targets in docstrings Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 5 +++-- tests/scripts/generate_bignum_tests.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index eb5c7631f796..d5d7e5f0014c 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -34,8 +34,9 @@ class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. - Derive directly from this class when adding new file Targets, setting - `target_basename`. + Child classes of this class represent an output file, and can be referred + to as file targets. These indicate where test cases will be written to for + all subclasses of the file target, which is set by `target_basename`. Attributes: count: Counter for test cases from this class. diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index d156f56f8f90..b4915d846072 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -6,11 +6,11 @@ Class structure: -Child classes of test_generation.BaseTarget (file Targets) represent a target +Child classes of test_generation.BaseTarget (file targets) represent an output file. These indicate where test cases will be written to, for all subclasses of -this Target. Multiple Target classes should not reuse a `target_basename`. +this target. Multiple file targets should not reuse a `target_basename`. -Each subclass derived from a file Target can either be: +Each subclass derived from a file target can either be: - A concrete class, representing a test function, which generates test cases. - An abstract class containing shared methods and attributes, not associated with a test function. An example is BignumOperation, which provides @@ -24,7 +24,7 @@ Adding test case generation for a function: A subclass representing the test function should be added, deriving from a -file Target such as BignumTarget. This test class must set/implement the +file target such as BignumTarget. This test class must set/implement the following: - test_function: the function name from the associated .function file. - test_name: a descriptive name or brief summary to refer to the test From 4ed94a4f7e9abf1848747310b8db8f4b074d69cd Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 16 Sep 2022 17:03:54 +0100 Subject: [PATCH 41/42] Use a script specific description in CLI help Previous changes used the docstring of the test_generation module, which does not inform a user about the script. Signed-off-by: Werner Lewis --- scripts/mbedtls_dev/test_generation.py | 4 ++-- tests/scripts/generate_bignum_tests.py | 4 ++-- tests/scripts/generate_psa_tests.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index d5d7e5f0014c..5de0b885c8a6 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -178,9 +178,9 @@ def generate_target(self, name: str, *target_args) -> None: test_cases = self.targets[name](*target_args) self.write_test_data_file(name, test_cases) -def main(args, generator_class: Type[TestGenerator] = TestGenerator): +def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator): """Command line entry point.""" - parser = argparse.ArgumentParser(description=__doc__) + parser = argparse.ArgumentParser(description=description) parser.add_argument('--list', action='store_true', help='List available targets and exit') parser.add_argument('targets', nargs='*', metavar='TARGET', diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index b4915d846072..ceafa4a489e6 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -233,6 +233,6 @@ class BignumAdd(BignumOperation): def result(self) -> str: return quote_str("{:x}".format(self.int_a + self.int_b)) - if __name__ == '__main__': - test_generation.main(sys.argv[1:]) + # Use the section of the docstring relevant to the CLI as description + test_generation.main(sys.argv[1:], "\n".join(__doc__.splitlines()[:4])) diff --git a/tests/scripts/generate_psa_tests.py b/tests/scripts/generate_psa_tests.py index 1f516c9928fe..867a6b459453 100755 --- a/tests/scripts/generate_psa_tests.py +++ b/tests/scripts/generate_psa_tests.py @@ -919,4 +919,4 @@ def generate_target(self, name: str, *target_args) -> None: super().generate_target(name, self.info) if __name__ == '__main__': - test_generation.main(sys.argv[1:], PSATestGenerator) + test_generation.main(sys.argv[1:], __doc__, PSATestGenerator) From 05998a00b1a6dff02c1334ee91020ae0ee12d6e0 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 21 Sep 2022 16:55:21 +0100 Subject: [PATCH 42/42] Add file generated by generate_bignum_tests.py Signed-off-by: Werner Lewis --- tests/suites/test_suite_mpi.generated.data | 381 +++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 tests/suites/test_suite_mpi.generated.data diff --git a/tests/suites/test_suite_mpi.generated.data b/tests/suites/test_suite_mpi.generated.data new file mode 100644 index 000000000000..947d06f7fb41 --- /dev/null +++ b/tests/suites/test_suite_mpi.generated.data @@ -0,0 +1,381 @@ +# Automatically generated by generate_bignum_tests.py. Do not edit! + +MPI add #1 0 (null) + 0 (null) +mbedtls_mpi_add_mpi:"":"":"0" + +MPI add #2 0 (null) + 0 (1 limb) +mbedtls_mpi_add_mpi:"":"0":"0" + +MPI add #3 0 (null) + positive +mbedtls_mpi_add_mpi:"":"7b":"7b" + +MPI add #4 0 (null) + negative +mbedtls_mpi_add_mpi:"":"-7b":"-7b" + +MPI add #5 0 (null) + positive with leading zero limb +mbedtls_mpi_add_mpi:"":"0000000000000000123":"123" + +MPI add #6 0 (null) + negative with leading zero limb +mbedtls_mpi_add_mpi:"":"-0000000000000000123":"-123" + +MPI add #7 0 (null) + large positive +mbedtls_mpi_add_mpi:"":"1230000000000000000":"1230000000000000000" + +MPI add #8 0 (null) + large negative +mbedtls_mpi_add_mpi:"":"-1230000000000000000":"-1230000000000000000" + +MPI add #9 0 (1 limb) + 0 (1 limb) +mbedtls_mpi_add_mpi:"0":"0":"0" + +MPI add #10 0 (1 limb) + positive +mbedtls_mpi_add_mpi:"0":"7b":"7b" + +MPI add #11 0 (1 limb) + negative +mbedtls_mpi_add_mpi:"0":"-7b":"-7b" + +MPI add #12 0 (1 limb) + positive with leading zero limb +mbedtls_mpi_add_mpi:"0":"0000000000000000123":"123" + +MPI add #13 0 (1 limb) + negative with leading zero limb +mbedtls_mpi_add_mpi:"0":"-0000000000000000123":"-123" + +MPI add #14 0 (1 limb) + large positive +mbedtls_mpi_add_mpi:"0":"1230000000000000000":"1230000000000000000" + +MPI add #15 0 (1 limb) + large negative +mbedtls_mpi_add_mpi:"0":"-1230000000000000000":"-1230000000000000000" + +MPI add #16 positive + positive +mbedtls_mpi_add_mpi:"7b":"7b":"f6" + +MPI add #17 positive + negative +mbedtls_mpi_add_mpi:"7b":"-7b":"0" + +MPI add #18 positive + positive with leading zero limb +mbedtls_mpi_add_mpi:"7b":"0000000000000000123":"19e" + +MPI add #19 positive + negative with leading zero limb +mbedtls_mpi_add_mpi:"7b":"-0000000000000000123":"-a8" + +MPI add #20 positive + large positive +mbedtls_mpi_add_mpi:"7b":"1230000000000000000":"123000000000000007b" + +MPI add #21 positive + large negative +mbedtls_mpi_add_mpi:"7b":"-1230000000000000000":"-122ffffffffffffff85" + +MPI add #22 negative + negative +mbedtls_mpi_add_mpi:"-7b":"-7b":"-f6" + +MPI add #23 negative + positive with leading zero limb +mbedtls_mpi_add_mpi:"-7b":"0000000000000000123":"a8" + +MPI add #24 negative + negative with leading zero limb +mbedtls_mpi_add_mpi:"-7b":"-0000000000000000123":"-19e" + +MPI add #25 negative + large positive +mbedtls_mpi_add_mpi:"-7b":"1230000000000000000":"122ffffffffffffff85" + +MPI add #26 negative + large negative +mbedtls_mpi_add_mpi:"-7b":"-1230000000000000000":"-123000000000000007b" + +MPI add #27 positive with leading zero limb + positive with leading zero limb +mbedtls_mpi_add_mpi:"0000000000000000123":"0000000000000000123":"246" + +MPI add #28 positive with leading zero limb + negative with leading zero limb +mbedtls_mpi_add_mpi:"0000000000000000123":"-0000000000000000123":"0" + +MPI add #29 positive with leading zero limb + large positive +mbedtls_mpi_add_mpi:"0000000000000000123":"1230000000000000000":"1230000000000000123" + +MPI add #30 positive with leading zero limb + large negative +mbedtls_mpi_add_mpi:"0000000000000000123":"-1230000000000000000":"-122fffffffffffffedd" + +MPI add #31 negative with leading zero limb + negative with leading zero limb +mbedtls_mpi_add_mpi:"-0000000000000000123":"-0000000000000000123":"-246" + +MPI add #32 negative with leading zero limb + large positive +mbedtls_mpi_add_mpi:"-0000000000000000123":"1230000000000000000":"122fffffffffffffedd" + +MPI add #33 negative with leading zero limb + large negative +mbedtls_mpi_add_mpi:"-0000000000000000123":"-1230000000000000000":"-1230000000000000123" + +MPI add #34 large positive + large positive +mbedtls_mpi_add_mpi:"1230000000000000000":"1230000000000000000":"2460000000000000000" + +MPI add #35 large positive + large negative +mbedtls_mpi_add_mpi:"1230000000000000000":"-1230000000000000000":"0" + +MPI add #36 large negative + large negative +mbedtls_mpi_add_mpi:"-1230000000000000000":"-1230000000000000000":"-2460000000000000000" + +MPI add #37 large positive + large positive +mbedtls_mpi_add_mpi:"1c67967269c6":"1c67967269c6":"38cf2ce4d38c" + +MPI add #38 large positive + positive +mbedtls_mpi_add_mpi:"1c67967269c6":"9cde3":"1c67967c37a9" + +MPI add #39 large positive + large negative +mbedtls_mpi_add_mpi:"1c67967269c6":"-1c67967269c6":"0" + +MPI add #40 large positive + negative +mbedtls_mpi_add_mpi:"1c67967269c6":"-9cde3":"1c6796689be3" + +MPI add #41 positive + positive +mbedtls_mpi_add_mpi:"9cde3":"9cde3":"139bc6" + +MPI add #42 positive + large negative +mbedtls_mpi_add_mpi:"9cde3":"-1c67967269c6":"-1c6796689be3" + +MPI add #43 positive + negative +mbedtls_mpi_add_mpi:"9cde3":"-9cde3":"0" + +MPI add #44 large negative + large negative +mbedtls_mpi_add_mpi:"-1c67967269c6":"-1c67967269c6":"-38cf2ce4d38c" + +MPI add #45 large negative + negative +mbedtls_mpi_add_mpi:"-1c67967269c6":"-9cde3":"-1c67967c37a9" + +MPI add #46 negative + negative +mbedtls_mpi_add_mpi:"-9cde3":"-9cde3":"-139bc6" + +MPI compare #1 0 (null) == 0 (null) +mbedtls_mpi_cmp_mpi:"":"":0 + +MPI compare #2 0 (null) == 0 (1 limb) +mbedtls_mpi_cmp_mpi:"":"0":0 + +MPI compare #3 0 (null) < positive +mbedtls_mpi_cmp_mpi:"":"7b":-1 + +MPI compare #4 0 (null) > negative +mbedtls_mpi_cmp_mpi:"":"-7b":1 + +MPI compare #5 0 (null) < positive with leading zero limb +mbedtls_mpi_cmp_mpi:"":"0000000000000000123":-1 + +MPI compare #6 0 (null) > negative with leading zero limb +mbedtls_mpi_cmp_mpi:"":"-0000000000000000123":1 + +MPI compare #7 0 (null) < large positive +mbedtls_mpi_cmp_mpi:"":"1230000000000000000":-1 + +MPI compare #8 0 (null) > large negative +mbedtls_mpi_cmp_mpi:"":"-1230000000000000000":1 + +MPI compare #9 0 (1 limb) == 0 (1 limb) +mbedtls_mpi_cmp_mpi:"0":"0":0 + +MPI compare #10 0 (1 limb) < positive +mbedtls_mpi_cmp_mpi:"0":"7b":-1 + +MPI compare #11 0 (1 limb) > negative +mbedtls_mpi_cmp_mpi:"0":"-7b":1 + +MPI compare #12 0 (1 limb) < positive with leading zero limb +mbedtls_mpi_cmp_mpi:"0":"0000000000000000123":-1 + +MPI compare #13 0 (1 limb) > negative with leading zero limb +mbedtls_mpi_cmp_mpi:"0":"-0000000000000000123":1 + +MPI compare #14 0 (1 limb) < large positive +mbedtls_mpi_cmp_mpi:"0":"1230000000000000000":-1 + +MPI compare #15 0 (1 limb) > large negative +mbedtls_mpi_cmp_mpi:"0":"-1230000000000000000":1 + +MPI compare #16 positive == positive +mbedtls_mpi_cmp_mpi:"7b":"7b":0 + +MPI compare #17 positive > negative +mbedtls_mpi_cmp_mpi:"7b":"-7b":1 + +MPI compare #18 positive < positive with leading zero limb +mbedtls_mpi_cmp_mpi:"7b":"0000000000000000123":-1 + +MPI compare #19 positive > negative with leading zero limb +mbedtls_mpi_cmp_mpi:"7b":"-0000000000000000123":1 + +MPI compare #20 positive < large positive +mbedtls_mpi_cmp_mpi:"7b":"1230000000000000000":-1 + +MPI compare #21 positive > large negative +mbedtls_mpi_cmp_mpi:"7b":"-1230000000000000000":1 + +MPI compare #22 negative == negative +mbedtls_mpi_cmp_mpi:"-7b":"-7b":0 + +MPI compare #23 negative < positive with leading zero limb +mbedtls_mpi_cmp_mpi:"-7b":"0000000000000000123":-1 + +MPI compare #24 negative > negative with leading zero limb +mbedtls_mpi_cmp_mpi:"-7b":"-0000000000000000123":1 + +MPI compare #25 negative < large positive +mbedtls_mpi_cmp_mpi:"-7b":"1230000000000000000":-1 + +MPI compare #26 negative > large negative +mbedtls_mpi_cmp_mpi:"-7b":"-1230000000000000000":1 + +MPI compare #27 positive with leading zero limb == positive with leading zero limb +mbedtls_mpi_cmp_mpi:"0000000000000000123":"0000000000000000123":0 + +MPI compare #28 positive with leading zero limb > negative with leading zero limb +mbedtls_mpi_cmp_mpi:"0000000000000000123":"-0000000000000000123":1 + +MPI compare #29 positive with leading zero limb < large positive +mbedtls_mpi_cmp_mpi:"0000000000000000123":"1230000000000000000":-1 + +MPI compare #30 positive with leading zero limb > large negative +mbedtls_mpi_cmp_mpi:"0000000000000000123":"-1230000000000000000":1 + +MPI compare #31 negative with leading zero limb == negative with leading zero limb +mbedtls_mpi_cmp_mpi:"-0000000000000000123":"-0000000000000000123":0 + +MPI compare #32 negative with leading zero limb < large positive +mbedtls_mpi_cmp_mpi:"-0000000000000000123":"1230000000000000000":-1 + +MPI compare #33 negative with leading zero limb > large negative +mbedtls_mpi_cmp_mpi:"-0000000000000000123":"-1230000000000000000":1 + +MPI compare #34 large positive == large positive +mbedtls_mpi_cmp_mpi:"1230000000000000000":"1230000000000000000":0 + +MPI compare #35 large positive > large negative +mbedtls_mpi_cmp_mpi:"1230000000000000000":"-1230000000000000000":1 + +MPI compare #36 large negative == large negative +mbedtls_mpi_cmp_mpi:"-1230000000000000000":"-1230000000000000000":0 + +MPI compare #37 negative > negative +mbedtls_mpi_cmp_mpi:"-2":"-3":1 + +MPI compare #38 negative == negative +mbedtls_mpi_cmp_mpi:"-2":"-2":0 + +MPI compare #39 positive < positive +mbedtls_mpi_cmp_mpi:"2b4":"2b5":-1 + +MPI compare #40 positive < positive +mbedtls_mpi_cmp_mpi:"2b5":"2b6":-1 + +MPI compare (abs) #1 0 (null) == 0 (null) +mbedtls_mpi_cmp_abs:"":"":0 + +MPI compare (abs) #2 0 (null) == 0 (1 limb) +mbedtls_mpi_cmp_abs:"":"0":0 + +MPI compare (abs) #3 0 (null) < positive +mbedtls_mpi_cmp_abs:"":"7b":-1 + +MPI compare (abs) #4 0 (null) < positive +mbedtls_mpi_cmp_abs:"":"7b":-1 + +MPI compare (abs) #5 0 (null) < positive with leading zero limb +mbedtls_mpi_cmp_abs:"":"0000000000000000123":-1 + +MPI compare (abs) #6 0 (null) < positive with leading zero limb +mbedtls_mpi_cmp_abs:"":"0000000000000000123":-1 + +MPI compare (abs) #7 0 (null) < large positive +mbedtls_mpi_cmp_abs:"":"1230000000000000000":-1 + +MPI compare (abs) #8 0 (null) < large positive +mbedtls_mpi_cmp_abs:"":"1230000000000000000":-1 + +MPI compare (abs) #9 0 (1 limb) == 0 (1 limb) +mbedtls_mpi_cmp_abs:"0":"0":0 + +MPI compare (abs) #10 0 (1 limb) < positive +mbedtls_mpi_cmp_abs:"0":"7b":-1 + +MPI compare (abs) #11 0 (1 limb) < positive +mbedtls_mpi_cmp_abs:"0":"7b":-1 + +MPI compare (abs) #12 0 (1 limb) < positive with leading zero limb +mbedtls_mpi_cmp_abs:"0":"0000000000000000123":-1 + +MPI compare (abs) #13 0 (1 limb) < positive with leading zero limb +mbedtls_mpi_cmp_abs:"0":"0000000000000000123":-1 + +MPI compare (abs) #14 0 (1 limb) < large positive +mbedtls_mpi_cmp_abs:"0":"1230000000000000000":-1 + +MPI compare (abs) #15 0 (1 limb) < large positive +mbedtls_mpi_cmp_abs:"0":"1230000000000000000":-1 + +MPI compare (abs) #16 positive == positive +mbedtls_mpi_cmp_abs:"7b":"7b":0 + +MPI compare (abs) #17 positive == positive +mbedtls_mpi_cmp_abs:"7b":"7b":0 + +MPI compare (abs) #18 positive < positive with leading zero limb +mbedtls_mpi_cmp_abs:"7b":"0000000000000000123":-1 + +MPI compare (abs) #19 positive < positive with leading zero limb +mbedtls_mpi_cmp_abs:"7b":"0000000000000000123":-1 + +MPI compare (abs) #20 positive < large positive +mbedtls_mpi_cmp_abs:"7b":"1230000000000000000":-1 + +MPI compare (abs) #21 positive < large positive +mbedtls_mpi_cmp_abs:"7b":"1230000000000000000":-1 + +MPI compare (abs) #22 positive == positive +mbedtls_mpi_cmp_abs:"7b":"7b":0 + +MPI compare (abs) #23 positive < positive with leading zero limb +mbedtls_mpi_cmp_abs:"7b":"0000000000000000123":-1 + +MPI compare (abs) #24 positive < positive with leading zero limb +mbedtls_mpi_cmp_abs:"7b":"0000000000000000123":-1 + +MPI compare (abs) #25 positive < large positive +mbedtls_mpi_cmp_abs:"7b":"1230000000000000000":-1 + +MPI compare (abs) #26 positive < large positive +mbedtls_mpi_cmp_abs:"7b":"1230000000000000000":-1 + +MPI compare (abs) #27 positive with leading zero limb == positive with leading zero limb +mbedtls_mpi_cmp_abs:"0000000000000000123":"0000000000000000123":0 + +MPI compare (abs) #28 positive with leading zero limb == positive with leading zero limb +mbedtls_mpi_cmp_abs:"0000000000000000123":"0000000000000000123":0 + +MPI compare (abs) #29 positive with leading zero limb < large positive +mbedtls_mpi_cmp_abs:"0000000000000000123":"1230000000000000000":-1 + +MPI compare (abs) #30 positive with leading zero limb < large positive +mbedtls_mpi_cmp_abs:"0000000000000000123":"1230000000000000000":-1 + +MPI compare (abs) #31 positive with leading zero limb == positive with leading zero limb +mbedtls_mpi_cmp_abs:"0000000000000000123":"0000000000000000123":0 + +MPI compare (abs) #32 positive with leading zero limb < large positive +mbedtls_mpi_cmp_abs:"0000000000000000123":"1230000000000000000":-1 + +MPI compare (abs) #33 positive with leading zero limb < large positive +mbedtls_mpi_cmp_abs:"0000000000000000123":"1230000000000000000":-1 + +MPI compare (abs) #34 large positive == large positive +mbedtls_mpi_cmp_abs:"1230000000000000000":"1230000000000000000":0 + +MPI compare (abs) #35 large positive == large positive +mbedtls_mpi_cmp_abs:"1230000000000000000":"1230000000000000000":0 + +MPI compare (abs) #36 large positive == large positive +mbedtls_mpi_cmp_abs:"1230000000000000000":"1230000000000000000":0 + +MPI compare (abs) #37 positive < positive +mbedtls_mpi_cmp_abs:"2":"3":-1 + +MPI compare (abs) #38 positive == positive +mbedtls_mpi_cmp_abs:"2":"2":0 + +MPI compare (abs) #39 positive < positive +mbedtls_mpi_cmp_abs:"2b4":"2b5":-1 + +MPI compare (abs) #40 positive < positive +mbedtls_mpi_cmp_abs:"2b5":"2b6":-1 + +# End of automatically generated file.