Merge pull request #9 from lindycoder/failures_output

Provide better failure output

src/asserts.sh

@@ -8,13 +8,13 @@ assert() {
 }
 
 _assert_equals() {
-    test "${1}" == "${2}" || fail "Expected <$2>\nGot:      <$1>"
+    test "${1}" == "${2}" || assertion_failed "Expected: <$2>\nGot:      <$1>"
 }
 
 _assert_succeeded() {
-    test ${1} -eq 0 || fail "Expected success exit code\nGot:      <$1>"
+    test ${1} -eq 0 || assertion_failed "Expected success exit code\nGot: <$1>"
 }
 
 _assert_failed() {
-    test ${1} -ne 0 || fail "Expected failure exit code\nGot:      <$1>"
+    test ${1} -ne 0 || assertion_failed "Expected failure exit code\nGot: <$1>"
 }

src/test-running.sh

@@ -34,13 +34,23 @@ if [ -z "${RUN_SINGLE_TEST:-""}" ]; then
     test_count=$((${test_count} + ${new_tests}))
   done
 
+  if [ -f ${registry}/failures_output ]; then
+    cat ${registry}/failures_output
+  fi
   echo ""
   echo "-------------------------"
   echo "Ran "$(_format_count ${test_count} "test")
   echo ""
-  echo ">>> SUCCESS <<<"
-  echo ""
-  exit 0
+  failure_count=$(cat ${registry}/failures_count)
+  if [ ${failure_count} -eq 0 ]; then
+    echo ">>> SUCCESS <<<"
+    echo ""
+    exit 0
+  else
+    echo ">>> FAILURE ("$(_format_count ${failure_count} "error")") <<<"
+    echo ""
+    exit 1
+  fi
 fi
 
 
@@ -85,6 +95,13 @@ _cleanup() {
 trap _cleanup INT TERM EXIT
 
 test_count=0
+failures=0
+
+assertion_failed() {
+    touch ${workspace}/.assertion_error
+    echo -e "$1"
+    return 1
+}
 
 for test in ${tests}; do
     _setup_workspace
@@ -95,15 +112,35 @@ for test in ${tests}; do
         ${setup}
     fi
 
-    printf $(_trim_test_prefix $(_file_base_name $(basename ${TEST_FILE}))).$(_trim_test_prefix ${test})...
-    ${test} || fail "Test failed with exit code $?"
-    echo "OK"
+    test_name=$(_trim_test_prefix $(_file_base_name $(basename ${TEST_FILE}))).$(_trim_test_prefix ${test})
+    printf ${test_name}...
+
+    failed=0
+
+    ${test} > ${workspace}/test_output || true
+
+    if [ ! -f ${workspace}/.assertion_error ]; then
+        echo "OK"
+    else
+        echo "FAILED"
+        failures=$((${failures} + 1))
+        cat >> ${REGISTRY}/failures_output <<FAILURE
+
+=========================
+FAIL: ${test_name}
+-------------------------
+$(cat ${workspace}/test_output)
+-------------------------
+FAILURE
+
+    fi
     test_count=$((${test_count} + 1))
 
     _cleanup
     popd >/dev/null
 done
 
 echo ${test_count} > ${REGISTRY}/test_count
+echo ${failures} > ${REGISTRY}/failures_count
 
 trap - INT TERM EXIT

test-fixtures/failing-test/src/fail.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exit 1

test-fixtures/failing-test/test/test_failing.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+test_that_fails() {
+    (bash fail.sh)
+    assert ${?} succeeded
+}

test/test_assert.sh

@@ -7,9 +7,19 @@ test_assert_equals_with_integers() {
 }
 
 test_assert_equals_with_integer_failing() {
-    (assert 1 equals 0)
-
-    assert ${?} failed
+    assert 1 equals 0 > assertion_output
+    result=${?}
+    rm .assertion_error # The test runner would think the test failed
+
+    assertion_error=$(cat assertion_output)
+
+    expected_error=$(cat <<-EXP
+Expected: <0>
+Got:      <1>
+EXP
+)
+    assert ${result} failed
+    assert "${assertion_error}" equals "${expected_error}"
 }
 
 test_assert_equals_with_strings_passing() {
@@ -19,9 +29,19 @@ test_assert_equals_with_strings_passing() {
 }
 
 test_assert_equals_with_strings_failing() {
-    (assert "no" equals "yes")
-
-    assert ${?} failed
+    assert "no" equals "yes" > assertion_output
+    result=${?}
+    rm .assertion_error # The test runner would think the test failed
+
+    assertion_error=$(cat assertion_output)
+
+    expected_error=$(cat <<-EXP
+Expected: <yes>
+Got:      <no>
+EXP
+)
+    assert ${result} failed
+    assert "${assertion_error}" equals "${expected_error}"
 }
 
 test_assert_multiworks_string_works() {

test/test_runner_output.sh

@@ -78,3 +78,35 @@ EXP
 )
     assert "${actual}" equals "${expected}"
 }
+
+test_output_is_fine_when_a_test_fails() {
+
+    cp -aR ${TEST_ROOT_DIR}/../test-fixtures/failing-test/* .
+
+    unset RUN_SINGLE_TEST
+    actual=$(${TEST_ROOT_DIR}/../target/sbtest.sh)
+    assert ${?} failed
+
+    expected=$(cat <<-EXP
+
+Running Simple Bash Tests
+-------------------------
+
+failing.that_fails...FAILED
+
+=========================
+FAIL: failing.that_fails
+-------------------------
+Expected success exit code
+Got: <1>
+-------------------------
+
+-------------------------
+Ran 1 test
+
+>>> FAILURE (1 error) <<<
+
+EXP
+)
+    assert "${actual}" equals "${expected}"
+}