Skip to content

Commit

Permalink
Merge pull request #7136 from yanrayw/5692-record-compatsh-test-cases
Browse files Browse the repository at this point in the history 
Record the outcome of each test case in compat.sh
  • Loading branch information
@daverodgman
daverodgman authored Jul 10, 2023
2 parents 5c41ae8 + 5b33f64 commit e183ece
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 32 deletions.
12 changes: 9 additions & 3 deletions docs/architecture/testing/test-framework.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,23 @@ Each test case has a description which succinctly describes for a human audience

#### SSL test case descriptions

Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_tests`.
Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_test`.

The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, the description must be written on the same line as `run_test`, in double quotes, for the sake of `check_test_cases.py`.

### SSL cipher suite tests

Each test case in `compat.sh` has a description which succinctly describes for a human audience what the test does. The test description is `$TITLE` defined in `run_client`.

The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, failure cause in `compat.sh` is not classified as `ssl-opt.sh`, so the information of failed log files are followed as prompt.

## Running tests

### Outcome file

#### Generating an outcome file

Unit tests and `ssl-opt.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file.
Unit tests, `ssl-opt.sh` and `compat.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file.

If you run `all.sh --outcome-file test-outcome.csv`, this collects the outcome of all the test cases in `test-outcome.csv`.

Expand All @@ -52,7 +58,7 @@ The outcome file has 6 fields:

* **Platform**: a description of the platform, e.g. `Linux-x86_64` or `Linux-x86_64-gcc7-msan`.
* **Configuration**: a unique description of the configuration (`mbedtls_config.h`).
* **Test suite**: `test_suite_xxx` or `ssl-opt`.
* **Test suite**: `test_suite_xxx`, `ssl-opt` or `compat`.
* **Test case**: the description of the test case.
* **Result**: one of `PASS`, `SKIP` or `FAIL`.
* **Cause**: more information explaining the result.
87 changes: 66 additions & 21 deletions tests/compat.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ set -u
# where it may output seemingly unlimited length error logs.
ulimit -f 20971520

ORIGINAL_PWD=$PWD
if ! cd "$(dirname "$0")"; then
exit 125
fi

# initialise counters
TESTS=0
FAILED=0
Expand Down Expand Up @@ -77,6 +82,17 @@ else
PEER_GNUTLS=""
fi

guess_config_name() {
if git diff --quiet ../include/mbedtls/mbedtls_config.h 2>/dev/null; then
echo "default"
else
echo "unknown"
fi
}
: ${MBEDTLS_TEST_OUTCOME_FILE=}
: ${MBEDTLS_TEST_CONFIGURATION:="$(guess_config_name)"}
: ${MBEDTLS_TEST_PLATFORM:="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}

# default values for options
# /!\ keep this synchronised with:
# - basic-build-test.sh
Expand Down Expand Up @@ -110,6 +126,8 @@ print_usage() {
printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n"
printf " -M|--memcheck\tCheck memory leaks and errors.\n"
printf " -v|--verbose\tSet verbose output.\n"
printf " --outcome-file\tFile where test outcomes are written\n"
printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
}

get_options() {
Expand Down Expand Up @@ -139,6 +157,9 @@ get_options() {
-M|--memcheck)
MEMCHECK=1
;;
--outcome-file)
shift; MBEDTLS_TEST_OUTCOME_FILE=$1
;;
-h|--help)
print_usage
exit 0
Expand Down Expand Up @@ -805,6 +826,39 @@ wait_client_done() {
echo "EXIT: $EXIT" >> $CLI_OUT
}

# record_outcome <outcome> [<failure-reason>]
record_outcome() {
echo "$1"
if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
# The test outcome file has the format (in single line):
# platform;configuration;
# test suite name;test case description;
# PASS/FAIL/SKIP;[failure cause]
printf '%s;%s;%s;%s;%s;%s\n' \
"$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \
"compat" "$TITLE" \
"$1" "${2-}" \
>> "$MBEDTLS_TEST_OUTCOME_FILE"
fi
}

# display additional information if test case fails
report_fail() {
FAIL_PROMPT="outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log"
record_outcome "FAIL" "$FAIL_PROMPT"
cp $SRV_OUT c-srv-${TESTS}.log
cp $CLI_OUT c-cli-${TESTS}.log
echo " ! $FAIL_PROMPT"

if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then
echo " ! server output:"
cat c-srv-${TESTS}.log
echo " ! ==================================================="
echo " ! client output:"
cat c-cli-${TESTS}.log
fi
}

# run_client PROGRAM_NAME STANDARD_CIPHER_SUITE PROGRAM_CIPHER_SUITE
run_client() {
# announce what we're going to do
Expand All @@ -817,7 +871,7 @@ run_client() {
# should we skip?
if [ "X$SKIP_NEXT" = "XYES" ]; then
SKIP_NEXT="NO"
echo "SKIP"
record_outcome "SKIP"
SKIPPED=$(( $SKIPPED + 1 ))
return
fi
Expand Down Expand Up @@ -911,26 +965,14 @@ run_client() {
# report and count result
case $RESULT in
"0")
echo PASS
record_outcome "PASS"
;;
"1")
echo SKIP
record_outcome "SKIP"
SKIPPED=$(( $SKIPPED + 1 ))
;;
"2")
echo FAIL
cp $SRV_OUT c-srv-${TESTS}.log
cp $CLI_OUT c-cli-${TESTS}.log
echo " ! outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log"

if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then
echo " ! server output:"
cat c-srv-${TESTS}.log
echo " ! ==================================================="
echo " ! client output:"
cat c-cli-${TESTS}.log
fi

report_fail
FAILED=$(( $FAILED + 1 ))
;;
esac
Expand All @@ -942,13 +984,16 @@ run_client() {
# MAIN
#

if cd $( dirname $0 ); then :; else
echo "cd $( dirname $0 ) failed" >&2
exit 1
fi

get_options "$@"

# Make the outcome file path relative to the original directory, not
# to .../tests
case "$MBEDTLS_TEST_OUTCOME_FILE" in
[!/]*)
MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
;;
esac

# sanity checks, avoid an avalanche of errors
if [ ! -x "$M_SRV" ]; then
echo "Command '$M_SRV' is not an executable file" >&2
Expand Down
19 changes: 11 additions & 8 deletions tests/ssl-opt.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,9 @@ get_options() {
-p|--preserve-logs)
PRESERVE_LOGS=1
;;
--outcome-file)
shift; MBEDTLS_TEST_OUTCOME_FILE=$1
;;
--port)
shift; SRV_PORT=$1
;;
Expand All @@ -200,14 +203,6 @@ get_options() {
done
}

# Make the outcome file path relative to the original directory, not
# to .../tests
case "$MBEDTLS_TEST_OUTCOME_FILE" in
[!/]*)
MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
;;
esac

# Read boolean configuration options from mbedtls_config.h for easy and quick
# testing. Skip non-boolean options (with something other than spaces
# and a comment after "#define SYMBOL"). The variable contains a
Expand Down Expand Up @@ -1791,6 +1786,14 @@ cleanup() {

get_options "$@"

# Make the outcome file path relative to the original directory, not
# to .../tests
case "$MBEDTLS_TEST_OUTCOME_FILE" in
[!/]*)
MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
;;
esac

populate_enabled_hash_algs

# Optimize filters: if $FILTER and $EXCLUDE can be expressed as shell
Expand Down

0 comments on commit e183ece

Please sign in to comment.