From f45a8eae3b07950706a6966877e566204c4c4237 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Mon, 20 Feb 2023 15:49:10 +0800 Subject: [PATCH 1/4] compat.sh: call record_fail if test case fails Signed-off-by: Yanray Wang --- tests/compat.sh | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 8f7d72c7b62a..a863cc17594a 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -790,6 +790,22 @@ wait_client_done() { echo "EXIT: $EXIT" >> $CLI_OUT } +# display additional information if test case fails +report_fail() { + echo "FAIL" + cp $SRV_OUT c-srv-${TESTS}.log + cp $CLI_OUT c-cli-${TESTS}.log + echo " ! outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log" + + if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then + echo " ! server output:" + cat c-srv-${TESTS}.log + echo " ! ===================================================" + echo " ! client output:" + cat c-cli-${TESTS}.log + fi +} + # run_client PROGRAM_NAME STANDARD_CIPHER_SUITE PROGRAM_CIPHER_SUITE run_client() { # announce what we're going to do @@ -903,19 +919,7 @@ run_client() { SKIPPED=$(( $SKIPPED + 1 )) ;; "2") - echo FAIL - cp $SRV_OUT c-srv-${TESTS}.log - cp $CLI_OUT c-cli-${TESTS}.log - echo " ! outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log" - - if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then - echo " ! server output:" - cat c-srv-${TESTS}.log - echo " ! ===================================================" - echo " ! client output:" - cat c-cli-${TESTS}.log - fi - + report_fail FAILED=$(( $FAILED + 1 )) ;; esac From ad470630027c314226177fcff4ceabc6fb920f88 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Mon, 20 Feb 2023 14:58:03 +0800 Subject: [PATCH 2/4] compat.sh: add support to record outcome of test cases If the environment variable MBEDTLS_TEST_OUTCOME_FILE is set, the test outcome file records each test case in a single line with the format of PLATFORM;CONFIGURATION;compat;TEST CASE DESCRIPTION;RESULT;[CAUSE] - CONFIGURATION comes from MBEDTLS_TEST_CONFIGURATION to record configuration of each test case - PLATFORM is either set by users or calculated from test platform - RESULT is one of PASS, FAIL or SKIP. If test case fails, srv_out/cli_out follows as FAILURE CAUSE. Signed-off-by: Yanray Wang --- tests/compat.sh | 61 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index a863cc17594a..5dd5f9e0fb98 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -30,6 +30,11 @@ set -u # where it may output seemingly unlimited length error logs. ulimit -f 20971520 +ORIGINAL_PWD=$PWD +if ! cd "$(dirname "$0")"; then + exit 125 +fi + # initialise counters TESTS=0 FAILED=0 @@ -77,6 +82,17 @@ else PEER_GNUTLS="" fi +guess_config_name() { + if git diff --quiet ../include/mbedtls/mbedtls_config.h 2>/dev/null; then + echo "default" + else + echo "unknown" + fi +} +: ${MBEDTLS_TEST_OUTCOME_FILE=} +: ${MBEDTLS_TEST_CONFIGURATION:="$(guess_config_name)"} +: ${MBEDTLS_TEST_PLATFORM:="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"} + # default values for options # /!\ keep this synchronised with: # - basic-build-test.sh @@ -110,6 +126,8 @@ print_usage() { printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n" printf " -M|--memcheck\tCheck memory leaks and errors.\n" printf " -v|--verbose\tSet verbose output.\n" + printf " --outcome-file\tFile where test outcomes are written\n" + printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" } get_options() { @@ -139,6 +157,9 @@ get_options() { -M|--memcheck) MEMCHECK=1 ;; + --outcome-file) + shift; MBEDTLS_TEST_OUTCOME_FILE=$1 + ;; -h|--help) print_usage exit 0 @@ -790,12 +811,29 @@ wait_client_done() { echo "EXIT: $EXIT" >> $CLI_OUT } +# record_outcome [] +record_outcome() { + echo "$1" + if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then + # The test outcome file has the format (in single line): + # platform;configuration; + # test suite name;test case description; + # PASS/FAIL/SKIP;[failure cause] + printf '%s;%s;%s;%s;%s;%s\n' \ + "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \ + "compat" "$TITLE" \ + "$1" "${2-}" \ + >> "$MBEDTLS_TEST_OUTCOME_FILE" + fi +} + # display additional information if test case fails report_fail() { - echo "FAIL" + FAIL_PROMPT="outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log" + record_outcome "FAIL" "$FAIL_PROMPT" cp $SRV_OUT c-srv-${TESTS}.log cp $CLI_OUT c-cli-${TESTS}.log - echo " ! outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log" + echo " ! $FAIL_PROMPT" if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then echo " ! server output:" @@ -818,7 +856,7 @@ run_client() { # should we skip? if [ "X$SKIP_NEXT" = "XYES" ]; then SKIP_NEXT="NO" - echo "SKIP" + record_outcome "SKIP" SKIPPED=$(( $SKIPPED + 1 )) return fi @@ -912,10 +950,10 @@ run_client() { # report and count result case $RESULT in "0") - echo PASS + record_outcome "PASS" ;; "1") - echo SKIP + record_outcome "SKIP" SKIPPED=$(( $SKIPPED + 1 )) ;; "2") @@ -931,13 +969,16 @@ run_client() { # MAIN # -if cd $( dirname $0 ); then :; else - echo "cd $( dirname $0 ) failed" >&2 - exit 1 -fi - get_options "$@" +# Make the outcome file path relative to the original directory, not +# to .../tests +case "$MBEDTLS_TEST_OUTCOME_FILE" in + [!/]*) + MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE" + ;; +esac + # sanity checks, avoid an avalanche of errors if [ ! -x "$M_SRV" ]; then echo "Command '$M_SRV' is not an executable file" >&2 From 7fc349e9037d5ca6e0419ee941813d5e2c10eb2e Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Mon, 20 Feb 2023 16:18:56 +0800 Subject: [PATCH 3/4] test-framework.md: document compat.sh Signed-off-by: Yanray Wang --- docs/architecture/testing/test-framework.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/architecture/testing/test-framework.md b/docs/architecture/testing/test-framework.md index 7780949e169b..80667df9204c 100644 --- a/docs/architecture/testing/test-framework.md +++ b/docs/architecture/testing/test-framework.md @@ -30,17 +30,23 @@ Each test case has a description which succinctly describes for a human audience #### SSL test case descriptions -Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_tests`. +Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_test`. The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, the description must be written on the same line as `run_test`, in double quotes, for the sake of `check_test_cases.py`. +### SSL cipher suite tests + +Each test case in `compat.sh` has a description which succinctly describes for a human audience what the test does. The test description is `$TITLE` defined in `run_client`. + +The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, failure cause in `compat.sh` is not classified as `ssl-opt.sh`, so the information of failed log files are followed as prompt. + ## Running tests ### Outcome file #### Generating an outcome file -Unit tests and `ssl-opt.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file. +Unit tests, `ssl-opt.sh` and `compat.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file. If you run `all.sh --outcome-file test-outcome.csv`, this collects the outcome of all the test cases in `test-outcome.csv`. @@ -52,7 +58,7 @@ The outcome file has 6 fields: * **Platform**: a description of the platform, e.g. `Linux-x86_64` or `Linux-x86_64-gcc7-msan`. * **Configuration**: a unique description of the configuration (`mbedtls_config.h`). -* **Test suite**: `test_suite_xxx` or `ssl-opt`. +* **Test suite**: `test_suite_xxx`, `ssl-opt` or `compat`. * **Test case**: the description of the test case. * **Result**: one of `PASS`, `SKIP` or `FAIL`. * **Cause**: more information explaining the result. From 5b33f6467b1dc7298191196eff496db996dad9e5 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Tue, 28 Feb 2023 11:56:59 +0800 Subject: [PATCH 4/4] ssl-opt.sh: support to parse --outcome-file Adjust where to set absolute path for MBEDTLS_TEST_OUTCOME_FILE as it's supposed to set its absolute path after all possible value assignment. Signed-off-by: Yanray Wang --- tests/ssl-opt.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index c176d0d62810..f892996ad425 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -177,6 +177,9 @@ get_options() { -p|--preserve-logs) PRESERVE_LOGS=1 ;; + --outcome-file) + shift; MBEDTLS_TEST_OUTCOME_FILE=$1 + ;; --port) shift; SRV_PORT=$1 ;; @@ -200,14 +203,6 @@ get_options() { done } -# Make the outcome file path relative to the original directory, not -# to .../tests -case "$MBEDTLS_TEST_OUTCOME_FILE" in - [!/]*) - MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE" - ;; -esac - # Read boolean configuration options from mbedtls_config.h for easy and quick # testing. Skip non-boolean options (with something other than spaces # and a comment after "#define SYMBOL"). The variable contains a @@ -1599,6 +1594,14 @@ cleanup() { get_options "$@" +# Make the outcome file path relative to the original directory, not +# to .../tests +case "$MBEDTLS_TEST_OUTCOME_FILE" in + [!/]*) + MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE" + ;; +esac + populate_enabled_hash_algs # Optimize filters: if $FILTER and $EXCLUDE can be expressed as shell