Skip to content

Commit

Permalink
Document the test scripts, and cosmetic changes
Browse files Browse the repository at this point in the history
  • Loading branch information
oSoMoN committed Feb 16, 2024
1 parent 5ccb762 commit 90d9826
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 5 deletions.
7 changes: 5 additions & 2 deletions tests/print-test-results.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
#!/bin/bash

# Print the test results written to a JSON file
# by run-upstream-testsuite.sh in a markdown format.
# Print the test results written to a JSON file by run-upstream-testsuite.sh
# in a markdown format. The printout includes the name of the test, the result,
# the URL to the test script and the contents of stdout and stderr.
# It can be used verbatim as the description when filing an issue for a test
# with an unexpected result.

json="test-results.json"
[[ -n $1 ]] && json="$1"
Expand Down
33 changes: 30 additions & 3 deletions tests/run-upstream-testsuite.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,28 @@
#!/bin/bash

# Run the GNU upstream test suite for diffutils against a local build of the
# Rust implementation, print out a summary of the test results, and writes a
# JSON file ('test-results.json') containing detailed information about the
# test run.

# The JSON file contains metadata about the test run, and for each test the
# result as well as the contents of stdout, stderr, and of all the files
# written by the test script, if any (excluding subdirectories).

# The script takes a shortcut to fetch only the test suite from the upstream
# repository and carefully avoids running the autotools machinery which is
# time-consuming and resource-intensive, and doesn't offer the option to not
# build the upstream binaries. As a consequence, the environment in which the
# tests are run might not match exactly that used when the upstream tests are
# run through the autotools.

# By default it expects a release build of the diffutils binary, but a
# different build profile can be specified as an argument
# (e.g. 'dev' or 'test').
# Unless overriden by the $TESTS environment variable, all tests in the test
# suite will be run. Tests targeting a command that is not yet implemented
# (e.g. cmp, diff3 or sdiff) are skipped.

scriptpath=$(dirname "$(readlink -f "$0")")
rev=$(git rev-parse HEAD)

Expand Down Expand Up @@ -59,7 +82,8 @@ for test in $tests
do
result="FAIL"
url="$urlroot$test?id=$upstreamrev"
# Run only the tests that invoke `diff`, because other binaries aren't implemented yet
# Run only the tests that invoke `diff`,
# because other binaries aren't implemented yet
if ! grep -E -s -q "(cmp|diff3|sdiff)" "$test"
then
sh "$test" 1> stdout.txt 2> stderr.txt && result="PASS" || exitcode=1
Expand Down Expand Up @@ -90,7 +114,10 @@ do
printf " %-40s $(tput setaf $color)$result$(tput sgr0)\n" "$test"
done
echo ""
echo "Summary: TOTAL: $total / $(tput setaf 2)PASS$normal: $passed / $(tput setaf 1)FAIL$normal: $failed / $(tput setaf 3)SKIP$normal: $skipped"
echo -n "Summary: TOTAL: $total / "
echo -n "$(tput setaf 2)PASS$normal: $passed / "
echo -n "$(tput setaf 1)FAIL$normal: $failed / "
echo "$(tput setaf 3)SKIP$normal: $skipped"
echo ""

json="\"tests\":[${json%,}]"
Expand All @@ -109,6 +136,6 @@ rm -rf "$tempdir"

resultsfile="test-results.json"
echo "$json" | jq > "$resultsfile"
echo "Results written to $resultsfile"
echo "Results written to $scriptpath/$resultsfile"

exit $exitcode

0 comments on commit 90d9826

Please sign in to comment.