assert option, max fail option & bugfixes (#13)

* init * syntax fix * bash sucks * test * fix * actual fix * fixing asserts * now finding failed * not getting error output * testing why fail count is off * forgot to update * more logging, better counting * should be checking now * syntax * echo debugging * regex somehow not working?? * better echoes, test fixing
croconut · Nov 7, 2021 · fd12020 · fd12020
1 parent f53d88e
commit fd12020
Show file tree

Hide file tree

Showing 3 changed files with 173 additions and 58 deletions.
diff --git a/.github/workflows/tester.yml b/.github/workflows/tester.yml
@@ -48,3 +48,29 @@ jobs:
         test-timeout: "45"
         minimum-pass: "0.7"
         direct-scene: "test/alt_mode/tests.tscn"
+    - uses: ./
+      with:
+        version: "3.2.2"
+        path: "tester"
+        test-timeout: "45"
+        minimum-pass: "0.7"
+        direct-scene: "test/alt_mode/tests.tscn"
+        assert-check: "true"
+    - uses: ./
+      continue-on-error: true
+      with:
+        version: "3.2.2"
+        path: "tester"
+        test-timeout: "45"
+        minimum-pass: "0.7"
+        assert-check: "false"
+        max-fails: "2"
+    - uses: ./
+      with:
+        version: "3.2.2"
+        path: "tester"
+        test-timeout: "45"
+        minimum-pass: "0.7"
+        direct-scene: "test/alt_mode/tests.tscn"
+        assert-check: "true"
+        max-fails: "6"
diff --git a/action.yml b/action.yml
@@ -37,6 +37,14 @@ inputs:
     description: "If instead of running the GUT command line you'd like to run a GUT scene instead: recommended to extend gut's plugin script and exit godot when all tests complete"
     required: false
     default: "false"
+  assert-check:
+    description: "Set to true to check by assert count instead of tests"
+    required: false
+    default: "false"
+  max-fails:
+    description: "Maximum number of failing asserts or tests, if this or pass rate fails then the action will fail. Is not checked if you do not set it"
+    required: false
+    default: "false"
 
 runs:
   using: 'docker'
@@ -51,6 +59,8 @@ runs:
   - ${{ inputs.minimum-pass }}
   - ${{ inputs.test-dir }}
   - ${{ inputs.direct-scene }}
+  - ${{ inputs.assert-check }}
+  - ${{ inputs.max-fails }}
 
 branding:
   icon: 'chevron-right'

diff --git a/entrypoint.sh b/entrypoint.sh
@@ -3,13 +3,6 @@
 
 set -e
 
-# credit: https://stackoverflow.com/questions/24283097/reusing-output-from-last-command-in-bash
-# capture the output of a command so it can be retrieved with ret
-cap () { tee /tmp/capture.out; }
-
-# return the output of the most recent command that was captured by cap
-ret () { cat /tmp/capture.out; }
-
 GODOT_VERSION=$1
 RELEASE_TYPE=$2
 PROJECT_DIRECTORY=$3
@@ -19,19 +12,131 @@ TEST_TIME=$6
 MINIMUM_PASSRATE=$7
 TEST_DIR=$8
 DIRECT_SCENE=$9
+# args above 9 require brackets
+ASSERT_CHECK=${10}
+MAX_FAILS=${11}
 
 GODOT_SERVER_TYPE="headless"
+TESTS=0
+FAILED=0
 CUSTOM_DL_PATH="~/custom_dl_folder"
 RUN_OPTIONS="-s addons/gut/gut_cmdln.gd -gdir=${TEST_DIR} -ginclude_subdirs -gexit"
 
-if [ "$RELEASE_TYPE" = "stable" ] ; then
+# credit: https://stackoverflow.com/questions/24283097/reusing-output-from-last-command-in-bash
+# capture the output of a command so it can be retrieved with ret
+cap() { tee /tmp/capture.out; }
+
+# return the output of the most recent command that was captured by cap
+ret() { cat /tmp/capture.out; }
+
+check_by_test() {
+
+    teststring="Tests:"
+    # new solution, need to count number of tests that were run e.g.
+    # a line that starts with "* test"
+    # versus the number of tests total
+    test_flagged="- test"
+    script_error="SCRIPT ERROR"
+
+    test_set=0
+    wait_for_fail=0
+    EXTRA_TESTS=0
+
+    while read line; do
+        # credit : https://stackoverflow.com/questions/17998978/removing-colors-from-output
+        temp=$(echo $line | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g')
+        # can see with below line all the extra characters that echo ignores
+        # echo LINE: $temp
+        if [[ $temp =~ ^$script_error ]]; then
+            FAILED=$((FAILED + 1))
+            EXTRA_TESTS=$FAILED
+            echo "script error found at $temp"
+            echo failed test count increased: $FAILED
+            echo
+            echo
+        elif [[ $temp =~ (Run)[[:space:]]+(Summary) ]]; then
+            test_set=1
+            echo reached test summary
+            echo
+            continue
+        fi
+
+        if [ "$test_set" -eq "0" ]; then
+            continue
+        elif [[ "$wait_for_fail" -eq "1" && $temp =~ ^[[:space:]]*(\[Failed\]) ]]; then
+            wait_for_fail=0
+            FAILED=$((FAILED + 1))
+            echo "test error found at $temp"
+            echo failed test count increased $FAILED
+            echo
+            echo
+        elif [[ $temp =~ ^$test_flagged ]]; then
+            wait_for_fail=1
+            echo "possible issue with test $temp ..."
+        fi
+
+        if [[ $temp =~ ^$teststring ]]; then
+            TESTS=${temp//[!0-9]/}
+            TESTS=$((TESTS + EXTRA_TESTS)) # adding script error fails that were found as additional failed tests
+            echo test count, including additional script error failures: $TESTS
+            echo
+            echo
+            break
+        fi
+
+    done \
+        <<<$(ret)
+}
+
+check_by_assert() {
+    script_error_fns=()
+
+    teststring="Tests:"
+    script_error="SCRIPT ERROR"
+
+    test_set=0
+
+    while read line; do
+        # credit : https://stackoverflow.com/questions/17998978/removing-colors-from-output
+        temp=$(echo $line | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g')
+        # can see with below line all the extra characters that echo ignores
+        # echo LINE: $temp
+        if [[ $temp =~ ^$script_error ]]; then
+            FAILED=$((FAILED + 1))
+            echo "script error found at $temp"
+            echo failed test count increased: $FAILED
+            echo
+            echo
+            continue
+        elif [[ $temp =~ ^$teststring ]]; then
+            test_set=1
+            continue
+        fi
+
+        if [ "$test_set" -eq "0" ]; then
+            continue
+        elif [[ $temp =~ ^[0-9]+[[:space:]]+(passed)[[:space:]]+[0-9]+[[:space:]]+(failed) ]]; then
+            passes=$(echo $temp | awk '{print $1}')
+            fails=$(echo $temp | awk '{print $3}')
+            FAILED=$((FAILED + fails))
+            TESTS=$((TESTS + FAILED + passes))
+            echo "total failed asserts $FAILED"
+            echo "total asserts $TESTS"
+            echo
+            echo
+            break
+        fi
+    done <<<$(ret)
+}
+
+if [ "$RELEASE_TYPE" = "stable" ]; then
     DL_PATH_SUFFIX=""
 else
     DL_PATH_SUFFIX="/${RELEASE_TYPE}"
 fi
 
 # if download places changes, will need updates to this if/else
-if [ "$IS_MONO" = "true" ] ; then
+if [ "$IS_MONO" = "true" ]; then
     GODOT_RELEASE_TYPE="${RELEASE_TYPE}_mono"
     DL_PATH_EXTENSION="${GODOT_VERSION}${DL_PATH_SUFFIX}/mono/"
     GODOT_EXTENSION="_64"
@@ -44,7 +149,7 @@ else
     FULL_GODOT_NAME=Godot_v${GODOT_VERSION}-${GODOT_RELEASE_TYPE}_linux_${GODOT_SERVER_TYPE}
 fi
 
-if [ "$DIRECT_SCENE" != "false" ] ; then
+if [ "$DIRECT_SCENE" != "false" ]; then
     RUN_OPTIONS="${DIRECT_SCENE}"
 fi
 
@@ -69,80 +174,54 @@ echo "running test suites ..."
 
 set +e
 # run tests
-if [ "$IS_MONO" = "true" ] ; then
-# need to init the imports
+if [ "$IS_MONO" = "true" ]; then
+    # need to init the imports
     timeout ${IMPORT_TIME} ./${CUSTOM_DL_PATH}/${FULL_GODOT_NAME}${GODOT_EXTENSION}/${FULL_GODOT_NAME}.64 -e
-    timeout ${TEST_TIME} ./${CUSTOM_DL_PATH}/${FULL_GODOT_NAME}${GODOT_EXTENSION}/${FULL_GODOT_NAME}.64 ${RUN_OPTIONS} | cap
+    timeout ${TEST_TIME} ./${CUSTOM_DL_PATH}/${FULL_GODOT_NAME}${GODOT_EXTENSION}/${FULL_GODOT_NAME}.64 ${RUN_OPTIONS} 2>&1 | cap
 else
     timeout ${IMPORT_TIME} ./${CUSTOM_DL_PATH}/${FULL_GODOT_NAME}${GODOT_EXTENSION} -e
-    timeout ${TEST_TIME} ./${CUSTOM_DL_PATH}/${FULL_GODOT_NAME}${GODOT_EXTENSION} ${RUN_OPTIONS} | cap
+    timeout ${TEST_TIME} ./${CUSTOM_DL_PATH}/${FULL_GODOT_NAME}${GODOT_EXTENSION} ${RUN_OPTIONS} 2>&1 | cap
 fi
 
 rm -rf ${CUSTOM_DL_PATH}/${FULL_GODOT_NAME}${GODOT_EXTENSION}
 rm -f ${CUSTOM_DL_PATH}/${FULL_GODOT_NAME}${GODOT_EXTENSION}.zip
 
 # parsing test output to fill test count and pass count variables
-TESTS=0
-FAILED=0
 
-script_error_fns=()
-
-teststring="Tests:"
-# new solution, need to count number of tests that were run e.g.
-# a line that starts with "* test"
-# versus the number of tests total
-test_failed_string="- test"
-script_error="SCRIPT ERROR"
-
-while read line; do
-    # credit : https://stackoverflow.com/questions/17998978/removing-colors-from-output
-    temp=$(echo $line | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g')
-    # can see with below line all the extra characters that echo ignores
-    # echo LINE: $temp
-    if [[ $temp =~ ^$script_error ]] ; then
-        FAILED=$((FAILED+1))
-        t_script_err_str=$(echo $temp | awk '{print $3}')
-        t_script_err_str=${t_script_err_str%?}
-        script_error_fns+=( $t_script_err_str )
-    elif [[ $temp =~ ^$teststring ]] ; then
-        TESTS=${temp//[!0-9]/}
-    elif [[ $temp =~ ^$test_failed_string ]] ; then
-        FAILED=$((FAILED+1))
-        match_fn_name=$(echo $temp | awk '{print $2}')
-        for i in "${script_error_fns[@]}" ; do
-            if [ "$i" == "$match_fn_name" ] ; then
-                FAILED=$((FAILED-1))
-                break
-            fi
-        done
-    fi
-done <<< `ret`
-
-# ensuring failing enough tests / being timed out cause failure for
-# the action
-echo "${outp0}"
-echo "${outp}"
+if [ "$ASSERT_CHECK" != "false" ]; then
+    check_by_assert
+else
+    check_by_test
+fi
 
 passrate=".0"
 endmsg=""
-if [ "$TESTS" -eq "0" ] ; then
+if [ "$TESTS" -eq "0" ]; then
     exitval=1
     endmsg="Tests failed due to timeout or there were no tests to run\n"
 else
-    passrate=`echo "scale=3; ($TESTS-$FAILED)/$TESTS"|bc -l`
+    passrate=$(echo "scale=3; ($TESTS-$FAILED)/$TESTS" | bc -l)
     echo -e "\n${passrate} pass rate\n"
-    if (( $(echo "$passrate >= $MINIMUM_PASSRATE" |bc -l) )); then
+    if (($(echo "$passrate >= $MINIMUM_PASSRATE" | bc -l))); then
         exitval=0
     else
         endmsg="Tests failed due to low passrate\n"
         exitval=1
     fi
+
+    if [[ "$MAX_FAILS" != "false" ]]; then
+        if [[ "$FAILED" -gt "$MAX_FAILS" ]]; then
+            endmsg="Tests failed due to fail count of ${FAILED} exceeding maximum of ${MAX_FAILS}"
+            exitval=1
+        fi
+    fi
 fi
 
-if [ "$endmsg" != "" ] ; then
-    echo -e "${endmsg}"
+if [ "$endmsg" != "" ]; then
     echo -e "Note: Tests may appear to pass on SCRIPT ERROR, but they were just ignored by GUT"
     echo -e "This action ensures those will fail"
+    echo
+    echo -e "${endmsg}"
 fi
 
 exit ${exitval}