test_all_examples

#!/bin/bash

trap 'kill 0;exit' INT QUIT ABRT KILL TERM

tmpFile=$(mktemp -t setlx-XXXXXXXXXX)

# set overrides for setlX launcher script
export OVERRIDE_setlXJarDirectory="$PWD/interpreter/"
export SETLX_LIBRARY_PATH="$PWD/interpreter/setlXlibrary"
export OVERRIDE_setlXjavaParameters="-d64 -Xmx6144m"
#export OVERRIDE_setlXjavaParameters="-d32 -Xmx3000m"

# other locations for tests
setlXlaunchScript="$PWD/interpreter/setlX"
setlXOptions="--predictableRandom --runtimeDebugging" # and --termLoop on most examples!
diffDir="$PWD"
exampleDirs=("example_SetlX_code" "Tutorial/Examples")

# be nice to the rest of the system
renice -n 10 $$

# prepare the interpreter
cd interpreter; ./runIntegrationTests >"$tmpFile" 2>&1; cd ..

# check if tests went fine
everythingOkCount=$(grep -c "Every test is fine. Move along sir!" "$tmpFile")

# alert the user and stop
if [ $everythingOkCount -ne 1 ]
  then
    echo "The initial integration tests failed!"
    rm -f "$tmpFile"
    exit 1
else
    echo -e "Integration Test successful!\n"
fi

# number of concurrent processes to start
numberOfThreads=$("$setlXlaunchScript" -e "nCPUs() \ 2")
if [ ! -z "$1" ]
  then
    if [[ "$1" =~ ^[1-9][0-9]*$ ]]
      then
        numberOfThreads="$1"
    else
        echo "First argument (number of threads) must be a number!"
        exit 1
    fi
fi

if [ "$numberOfThreads" -lt 1 ]
  then
    numberOfThreads="1"
fi

rm -f test_failure.*.result

# discover all example files
examples=()
for directory in "${exampleDirs[@]}"
  do
    while read -r -d $'\0' path
      do
        examples=("${examples[@]}" "$path")
    done < <( find -L "$directory" -name "*.stlx" -print0 )
done

# sort example files into automatic and tests
examplesAutomaticTest=()
examplesManualTest=()
for path in "${examples[@]}"
  do
    directoryname="$(dirname $path)"

    if [ "$setlX_block_performance" == "1" -a "$directoryname" == "example_SetlX_code/performance_test_code" ]
      then
        continue
    fi

    if [ ! -e "$path.reference" -a ! -e "$path.noTermLoop.reference" ]
      then
        examplesManualTest=("${examplesManualTest[@]}" "$path")
    else
        examplesAutomaticTest=("${examplesAutomaticTest[@]}" "$path")
    fi
done

# number of examples that can be tested automatically
count="${#examplesAutomaticTest[@]}"

# loop through all SetlX files, execute them and compare to reference result
started=0
threadID=0
pids=( )
references=( )
outputFiles=( )
filenames=( )
pathnames=( )
messages=( )
failedTests=()
for path in "${examplesAutomaticTest[@]}"
  do
    directoryname="$(dirname $path)"
    filename="$(basename $path)"
    outputFile=$(mktemp -t setlx-XXXXXXXXXX)

    started=$(bc <<< "$started + 1")
    echo -e "Starting test $started/$count.\n"
    message="Test of file '$path' ($started/$count)"

    if [ -e "$path.noTermLoop.reference" ]
      then
        referenceFile="$path.noTermLoop.reference"
        setlXOpts="$setlXOptions"
    else
        referenceFile="$path.reference"
        setlXOpts="$setlXOptions --termLoop"
    fi

    (
        # execute with setlX
        if [ -e "$path.input" ]
          then
            cd "$directoryname"; cat "$filename.input" | "$setlXlaunchScript" $setlXOpts "$filename" > "$outputFile" 2>&1; cd - >/dev/null
        else
            cd "$directoryname"; "$setlXlaunchScript" $setlXOpts "$filename" > "$outputFile" 2>&1; cd - >/dev/null
        fi
    )&
    pids[$threadID]=$!
    references[$threadID]="$referenceFile"
    outputFiles[$threadID]="$outputFile"
    filenames[$threadID]="$filename"
    pathnames[$threadID]="$path"
    messages[$threadID]="$message"

    # get next free ID in array, which is smaller as $numberOfThreads
    threadID=-1
    while [ $threadID -lt 0 ]
      do
        if [ $started -ge $count ]
          then
            wait
        fi
        for (( i = 0; i < $numberOfThreads; i++))
          do
            if [ $(kill -0 "${pids[$i]}" > /dev/null 2>&1; echo $?) -ne 0 ]
              then
                threadID=$i

                # diff result of finished thread
                if [ -f "${outputFiles[$i]}" ]
                  then
                    message=""

                    diff "${references[$i]}" "${outputFiles[$i]}" > "$tmpFile"
                    if [ $? -eq 1 ]
                      then

                        failedTests=("${failedTests[@]}" "${pathnames[$i]}")

                        outputName="test_failure.${pathnames[$i]}.result"
                        outputName="${outputName//\//.}"
                        mv "$tmpFile" "$diffDir/$outputName"
                        message="${messages[$i]} failed ... (see diff.${filenames[$i]}.result)\n"

                    else
                        message="${messages[$i]} successful!\n"
                    fi
                    echo -e "$message"

                    rm -f "${outputFiles[$i]}"
                fi
            fi
        done
        if [ $threadID -lt 0 ]
          then
            sleep 0.1
        fi
    done
done

rm -f "$tmpFile"

echo "The following examples need to be tested manually:"
for path in "${examplesManualTest[@]}"
  do
    echo $path
done


if [ "${#failedTests[@]}" -gt "0" ]
  then
    echo
    echo "The following examples failed:"
    for path in "${failedTests[@]}"
      do
        echo $path
    done
    echo
    echo "!! ${#failedTests[@]} examples did fail this test !!"
    echo
    "$setlXlaunchScript" --version
    exit 1
else
    echo
    echo "Everything is fabulous. Well done sir!"
    echo
    "$setlXlaunchScript" --version
    exit 0
fi