exec-tests

#!/bin/bash

# Test execution helper for tox.  It is not intended to be executed outside of a
# tox environment test execution.

# For now we ignore the tox environment directory argument.
_toxenvdir="${1}"
if [[ -z "${_toxenvdir}" ]]; then
    printf -- "Missing required tox environment directory.\n" >&2
    exit 2
fi
shift

# The first argument will be which major sub-system to run tests for: the
# agent code, the functional test client code, or the server code.

major="${1:-all}"
shift

if [[ "${major}" != "agent" && "${major}" != "server" && "${major}" != "client" && "${major}" != "all" ]]; then
    printf -- "Expected major sub-system to be 'agent', 'client', 'server', or 'all', got '%s'\n" "${major}" >&2
    exit 2
fi
if [[ "${major}" != "all" ]]; then
    major_list="${major}"
else
    major_list="agent client server"
fi

# Sub-test of the major test.
subtst="${1}"
shift

# Remaining positional arguments passed along to whatever test is run.
posargs="${@}"

# List of available agent sub-tests.
_agent_subtst_list="tool-scripts/datalog tool-scripts/postprocess tool-scripts util-scripts bench-scripts"

# The DEBUG_EXEC_UNITTESTS variable can be set to cause the particular
# command line actions taken to be echoed to stdout instead of actually
# executed.  This helps in debugging the operation of this script.
_ECHO=${DEBUG_EXEC_UNITTESTS:+echo}

parallel=${PBENCH_UNITTEST_PARALLEL:-auto}
if [[ "${parallel}" == "serial" ]]; then
    para_jobs_arg="-j 1"
    pytest_jobs_arg=""
elif [[ "${parallel}" == "auto" ]]; then
    para_jobs_arg=""
    pytest_jobs_arg="-n auto"
else
    printf -- "Unrecognized PBENCH_UNITTEST_PARALLEL environment variable value, '%s'\n" "${parallel}" >&2
    exit 2
fi

function _time {
    /usr/bin/time --format="\n\nCommand: '%C'\nExit status: %x\nTimings: user %Us, system %Ss, elapsed %es (%E, %P)\nMemory: max RSS %Mk, minor pf: %R, major pf: %F, swaps %W\nContext switches: inv %c, vol %w, signals %k\nI/O: fs in %I, fs out %O, socket in %r, socket out %s\n" ${@}
}

function run_legacy {
    local _rc
    local _major
    local _subtst

    _major=${1}
    shift
    _subtst=${1}
    shift

    printf -- "\n\n\nRunning %s/%s legacy unit tests\n\n" "${_major}" "${_subtst}"
    _time ${_major}/${_subtst}/unittests ${@}
    _rc=${?}
    if [[ ${_rc} -ne 0 ]]; then
        printf -- "\n%s %s legacy unit tests failed with '%s'\n\n" "${_major^}" "${_subtst}" "${_rc}"
    else
        printf -- "\n%s %s legacy unit tests succeeded\n\n" "${_major^}" "${_subtst}"
    fi
    return ${_rc}
}

function para_run_legacy {
    # When we want to use the `run_legacy` function with `parallel`, we have a
    # problem because the command line arguments are passed as one long
    # string.  So this jump function breaks up the arguments and invokes
    # `run_legacy` as expected.
    run_legacy ${1}
}

# Export functions for use with `parallel` below.
export -f para_run_legacy run_legacy _time

function verify_make_source_tree {
    # The agent and server-side sub-trees have individual ways to create the
    # source tree for a tar ball.  This function executes that step for the
    # given sub-tree.
    #
    # Arguments:
    #     _major -- Name of the sub-tree to execute the "make install"
    #
    # Returns the exit status of the "make install" execution, or the exit
    # status dependent command steps (e.g. make, cd, etc.).
    local _major=${1}
    local _rc

    if [[ ! -d ${_toxenvdir}/src ]]; then
        mkdir ${_toxenvdir}/src || return ${?}
    fi
    local _dir=${_toxenvdir}/src/${_major}
    rm -rf ${_dir}
    mkdir ${_dir} || return ${?}

    printf -- "\n\n\nVerifying %s source tree build\n\n" "${_major^}"
    (cd ${_major} && make DESTDIR=${_dir} install > ${_dir}.log 2>&1)
    _rc=${?}
    if [[ ${_rc} -ne 0 ]]; then
        cat ${_dir}.log
        printf -- "\n%s source tree build failed with '%s'\n\n" "${_major^}" "${_rc}"
    else
        printf -- "\n%s source tree build succeeded\n\n" "${_major^}"
        rm -rf ${_dir} ${_dir}.log
    fi
    return ${_rc}
}

if [[ -n "${COV_REPORT_XML}" ]]; then
    _cov_report_kind="xml"
    _cov_report_name="cov/report.xml"
else
    _cov_report_kind="html"
    _cov_report_name="${_toxenvdir}/cov/html"
fi
_cov_report="${_cov_report_kind}:${_cov_report_name}"

#+
# Test execution happens below.  Everything before this point should be for
# definition (functions or variables) and other required setup tasks.
#-

rc=0

if [[ "${subtst:-python}" == "python" ]]; then
    no_tests_specified=1
    for word in ${posargs}; do
        if [[ ${word} != -* ]]; then no_tests_specified=0; fi
    done
    if [[ ${no_tests_specified} -eq 1 ]]; then
        _pytest_majors="pbench.test.unit.common"
        for _major in ${major_list}; do
            _pytest_majors="${_pytest_majors} pbench.test.unit.${_major}"
            if [[ "${_major}" == "agent" ]]; then
                # TODO: real functional tests require a deployed instance. Current
                # agent "functional" tests are mocked Click tests rather than true
                # "functional" tests.
                _pytest_majors="${_pytest_majors} pbench.test.functional.agent"
            fi
        done
    fi

    printf -- "\n\n\nRunning %s python3-based unit tests via pytest\n\n" "${major_list// /,}"
    _pbench_sources=$(python3 -c 'import inspect, pathlib, pbench; print(pathlib.Path(inspect.getsourcefile(pbench)).parent)')
    # We use SQLALCHEMY_SILENCE_UBER_WARNING here to silence very prolific
    # warnings posted by SQLAlchemy 1.4.x about features / behaviors being
    # used which are not compatible with SQLAlchemy 2.x. Since we are not
    # going to switch to 2.x any time soon, we use the big hammer approach
    # to avoid the noise.
    SQLALCHEMY_SILENCE_UBER_WARNING=1 PYTHONUNBUFFERED=True _PBENCH_COV_DIR="${_toxenvdir}/cov" ${_ECHO} _time pytest \
        --tb=native \
        ${pytest_jobs_arg} \
        --basetemp="${_toxenvdir}/tmp" \
        --cov=${_pbench_sources} \
        --cov-report ${_cov_report} \
        -rs \
        --pyargs \
        ${posargs} \
        ${_pytest_majors}
    rc=${?}
    if [[ ${rc} -ne 0 ]]; then
        printf -- "\n%s pytest command failed with '%s'\n\n" "${major_list// /,}" "${rc}"
    else
        printf -- "\n%s pytest command succeeded\n\n" "${major_list// /,}"
    fi

    if [[ "${_cov_report_kind}" == "xml" && -n "${WORKSPACE}" ]]; then
        # For the Jenkins CI we need to adjust the generated XML report to
        # use WORKSPACE relative file name paths in the report.
        sed -i "s;${_pbench_sources};${WORKSPACE}/lib/pbench;" ${_cov_report_name}
    fi
fi

if [[ ( "${major}" == "all" || "${major}" == "agent" ) && ( "${subtst}" != "python" && "${subtst}" != "functional" ) ]]; then
    _para_jobs_file="${_toxenvdir}/agent-legacy-jobs.lis"
    trap "rm -f ${_para_jobs_file}" EXIT INT TERM

    for st in ${_agent_subtst_list}; do
        st_name=$(basename ${st})
        if [[ "${subtst}" == "legacy" || "${subtst:-${st_name}}" == "${st_name}" ]]; then
            echo "agent ${st} ${posargs}"
        fi
    done > ${_para_jobs_file}
    if [[ -s ${_para_jobs_file} ]]; then
        if [[ -n ${_ECHO} ]]; then
            cat ${_para_jobs_file}
        fi
        # The parallel program is really cool.  The usage of `parallel` is
        # internal and automated; only test code depends on this tool, and we,
        # as developers, have viewed the citation and are justified in
        # suppressing future displays of it in our development processes (use of
        # --will-cite below).
        ${_ECHO} parallel --will-cite -k --lb -a ${_para_jobs_file} ${para_jobs_arg} para_run_legacy
        if [[ ${?} -ne 0 ]]; then
            rc=1
        fi
    elif [[ "${subtst:-srcbld}" != "srcbld" ]]; then
        printf -- "Error - unrecognized sub-test, '%s'\n" "${subtst}" >&2
        rc=1
    fi

    # Verify the Agent Makefile source build functions correctly.
    if [[ "${subtst:-srcbld}" == "srcbld" && ${rc} -eq 0 ]]; then
        ${_ECHO} verify_make_source_tree agent || rc=1
    fi
fi

if [[ "${major}" == "all" || "${major}" == "server" ]]; then
    if [[ "${subtst}" == "legacy" ]]; then
        # Emit this warning since `legacy` was explicitly requested.
        printf -- "Warning - server legacy tests are no longer executed\n" >&2
        rc=1
    fi

    # Verify the Server Makefile source build functions correctly.
    if [[ "${subtst:-srcbld}" == "srcbld" && ${rc} -eq 0 ]]; then
        ${_ECHO} verify_make_source_tree server || rc=1
    fi

    # Run the server functional tests
    if [[ "${subtst}" == "functional" && ${rc} -eq 0 ]]; then
        server_arg=${1}
        shift
        posargs="${@}"
        # We use SQLALCHEMY_SILENCE_UBER_WARNING here ... (see above).
        SQLALCHEMY_SILENCE_UBER_WARNING=1 PYTHONUNBUFFERED=True PBENCH_SERVER=${server_arg} pytest --tb=native -v -s -rs --pyargs ${posargs} pbench.test.functional.server
        rc=${?}
    fi
fi

exit ${rc}