unittest.subr

# Copyright 2014 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# * Neither the name of Google Inc. nor the names of its contributors
#   may be used to endorse or promote products derived from this software
#   without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

shtk_import cli
shtk_import list


# Current expectation of the test regarding failure conditions.
_Shtk_Unittest_ExpectFailure=no


# Names of all the registered test cases.
#
# Upon program startup, this variable tracks all standalone test cases.
# However, when we process fixtures, this starts tracking the test cases within
# the currently-running fixture instead.
_Shtk_Unittest_TestCases=


# Names of all the registered test fixtures.
_Shtk_Unittest_TestFixtures=


# Functions to bring into the global namespace within the tests.
_Shtk_Unittest_Use="delayed_fail fail set_expect_failure skip"


# Hooks that can be overriden by the user.
one_time_setup() { true; }
one_time_teardown() { true; }


# Prepares for the execution of the given fully-qualified test case.
#
# This involves bringing all the required functions into the current namespace,
# creating a work directory for the test and entering the work directory.
#
# This function must be called from the subprocess used to run the test.
#
# The parent process must use _shtk_unittest_leave_test to clean up after
# execution.
_shtk_unittest_enter_test() {
    local name="${1}"; shift

    shtk_cli_info "Testing ${name}..."

    _Shtk_Unittest_ExpectFailure=no

    for function_name in ${_Shtk_Unittest_Use}; do
        eval "${function_name}() { " \
            "shtk_unittest_${function_name} \"\${@}\"; }"
    done

    mkdir "${name}"
    cd "${name}"

    HOME="$(pwd)"; export HOME
    TMPDIR="$(pwd)"; export TMPDIR
}


# Cleans up after executing a test case and computes its result.
#
# This function must be called from the parent process of a process that
# previously called _shtk_unittest_enter_test.
#
# The name parameter provides the fully-qualified name of the test case to
# clean up and the exit_code parameter holds the exit code of the
# subprocess used to run the test.
#
# Returns true if the test case exited successfully; false otherwise.
_shtk_unittest_leave_test() {
    local name="${1}"; shift
    local exit_code="${1}"; shift

    local delayed_failures=0
    [ ! -f "${name}/result.delayed-fail" ] \
        || delayed_failures="$(cat "${name}/result.delayed-fail")"

    local expected_failure=no
    [ ! -f "${name}/result.expect-fail" ] || expected_failure=yes

    local skipped=no
    [ ! -f "${name}/result.skipped" ] || skipped=yes

    local result
    if [ "${expected_failure}" = no ]; then
        if [ ${exit_code} -eq 0 ]; then
            if [ "${delayed_failures}" -gt 0 ]; then
                result=delayed-fail
            elif [ "${skipped}" = yes ]; then
                result=skip
            else
                result=pass
            fi
        else
            result=fail
        fi
    else
        if [ ${exit_code} -eq 0 ]; then
            if [ "${delayed_failures}" -gt 0 ]; then
                result=expected-failure
            else
                shtk_cli_warning "Expected failure but none found"
                result=fail
            fi
        else
            result=expected-failure
        fi
    fi
    rm -rf "${name}"

    case "${result}" in
        delayed-fail)
            shtk_cli_warning "Testing ${name}... FAILED" \
                "(${delayed_failures} delayed failures)"
            return 1
            ;;
        expected-failure)
            shtk_cli_info "Testing ${name}... EXPECTED FAILURE"
            return 0
            ;;
        fail)
            shtk_cli_warning "Testing ${name}... FAILED"
            return 1
            ;;
        pass)
            shtk_cli_info "Testing ${name}... PASSED"
            return 0
            ;;
        skip)
            shtk_cli_warning "Testing ${name}... SKIPPED"
            return 0
            ;;
    esac
    shtk_cli_error "Failed to determine test result"
}


# Defines assert and expect functions for a given name.
#
# For a given _shtk_unittest_check_${name} function, a pair of
# shtk_unittest_assert_${name} and shtk_unittest_expect_${name} functions are
# defined.  The former implements a fatal test and the latter a delayed test.
#
# A _shtk_unittest_check_${name} function must exist.  This function must
# take the name of the wrapper function as the first argument, the name of a
# "fail function" as the second argument, and then all of the check-specific
# arguments (including, possibly, options).
_shtk_unittest_register_check() {
    local name="${1}"; shift

    eval "shtk_unittest_assert_${name}() {
        _shtk_unittest_check_${name} shtk_unittest_assert_${name} \
            shtk_unittest_fail \"\${@}\"
    }"

    eval "shtk_unittest_expect_${name}() {
        _shtk_unittest_check_${name} shtk_unittest_expect_${name} \
            shtk_unittest_delayed_fail \"\${@}\"
    }"

    _Shtk_Unittest_Use="${_Shtk_Unittest_Use} assert_${name} expect_${name}"
}


# Executes a test case within a fixture.
#
# Returns true if the test case passes; false otherwise.
_shtk_unittest_run_fixture_test() {
    local fixture="${1}"; shift
    local basename="${1}"; shift

    local name="${fixture}__${basename}"

    shtk_list_contains "${basename}" ${_Shtk_Unittest_TestCases} \
        || shtk_cli_error "Attempting to run unregistered test case ${name}"

    (
        _shtk_unittest_enter_test "${name}"

        local failed=no
        if { setup; true; }; then
            ( "${basename}_test"; true ) || failed=yes
            { teardown; true; } || failed=yes
        else
            failed=yes
        fi
        [ "${failed}" = no ]
    )
    _shtk_unittest_leave_test "${name}" "${?}"
}


# Executes a standalone test case.
#
# Returns true if the test case passes; false otherwise.
_shtk_unittest_run_standalone_test() {
    local name="${1}"; shift

    shtk_list_contains "${name}" ${_Shtk_Unittest_TestCases} \
        || shtk_cli_error "Attempting to run unregistered test case ${name}"

    (
        _shtk_unittest_enter_test "${name}"

        "${name}_test"
        true
    )
    _shtk_unittest_leave_test "${name}" "${?}"
}


shtk_unittest_add_fixture() {
    local name="${1}"; shift

    shtk_list_contains "${name}" ${_Shtk_Unittest_TestFixtures} \
        && shtk_cli_error "Duplicate test fixture found: ${name}"

    eval "${name}_fixture() { shtk_cli_error '${name}_fixture not defined'; }"

    _Shtk_Unittest_TestFixtures="${_Shtk_Unittest_TestFixtures} ${name}"
}


shtk_unittest_add_test() {
    local name="${1}"; shift

    shtk_list_contains "${name}" ${_Shtk_Unittest_TestCases} \
        && shtk_cli_error "Duplicate test case found: ${name}"

    eval "${name}_test() { shtk_cli_error '${name}_test not defined'; }"

    _Shtk_Unittest_TestCases="${_Shtk_Unittest_TestCases} ${name}"
}


shtk_unittest_delayed_fail() {
    [ "${_Shtk_Unittest_ExpectFailure}" = no ] \
        || shtk_cli_error "delayed_fail does not support expected failures"

    # TODO(jmmv): Consider using ${LINENO} if available.
    shtk_cli_warning "Delayed failure: ${@}"

    # Because the test case runs in a subshell but we need to recover the number
    # of delayed failures from the main process, we have to persist this
    # information in the file system.
    local count=
    if [ -f result.delayed-fail ]; then
        count="$(cat result.delayed-fail)"
    else
        count=0
    fi
    echo "$((${count} + 1))" >result.delayed-fail || \
        shtk_cli_error "Failed to record delayed failure"
}


shtk_unittest_fail() {
    # TODO(jmmv): Consider using ${LINENO} if available.
    if [ "${_Shtk_Unittest_ExpectFailure}" = no ]; then
        shtk_cli_error "${@}"
    else
        shtk_cli_error "Expected failure:" "${@}"
    fi
}


shtk_unittest_main() {
    shtk_cli_set_help_command "man 1 shtk_unittest"

    local filters=
    local OPTIND
    while getopts :t: arg "${@}"; do
        case "${arg}" in
            t)  # Test filter.
                filters="${filters} ${OPTARG}"
                ;;

            :)
                shtk_cli_usage_error "Missing argument to option -${OPTARG}"
                ;;

            \?)
                shtk_cli_usage_error "Unknown option -${OPTARG}"
                ;;
        esac
    done
    shift $((${OPTIND} - 1))
    OPTIND=1  # Should not be necessary due to the 'local' above.

    [ "${#}" -eq 0 ] || shtk_cli_usage_error "No command-line arguments allowed"

    [ -n "${_Shtk_Unittest_TestFixtures}" -o \
      -n "${_Shtk_Unittest_TestCases}" ] \
        || shtk_cli_error "No test cases defined; did you" \
        "call shtk_unittest_add_fixture or shtk_unittest_add_test?"

    local passed=0
    local failed=0

    one_time_setup || shtk_cli_error "Failed to set up test program;" \
        "one_time_setup failed"

    for name in ${_Shtk_Unittest_TestCases}; do
        [ -z "${filters}" ] || shtk_list_contains "${name}" ${filters} \
            || continue

        if _shtk_unittest_run_standalone_test "${name}"; then
            passed=$((${passed} + 1))
        else
            failed=$((${failed} + 1))
        fi
    done

    for fixture in ${_Shtk_Unittest_TestFixtures}; do
        _Shtk_Unittest_TestCases=

        setup() { true; }
        teardown() { true; }
        eval ${fixture}_fixture

        for name in ${_Shtk_Unittest_TestCases}; do
            [ -z "${filters}" ] \
                || shtk_list_contains "${fixture}__${name}" ${filters} \
                || continue

            if _shtk_unittest_run_fixture_test "${fixture}" "${name}"; then
                passed=$((${passed} + 1))
            else
                failed=$((${failed} + 1))
            fi
        done
    done

    one_time_teardown || shtk_cli_error "Failed to clean up test program;" \
        "one_time_teardown failed"

    local total=$((${passed} + ${failed}))
    [ ${total} -gt 0 ] || shtk_cli_error "No tests were run"
    if [ ${failed} -eq 0 ]; then
        shtk_cli_info "Ran ${total} tests; ALL PASSED"
        return 0
    else
        shtk_cli_warning "Ran ${total} tests; ${failed} FAILED"
        return 1
    fi
}


shtk_unittest_set_expect_failure() {
    _Shtk_Unittest_ExpectFailure=yes
    touch result.expect-fail
}


shtk_unittest_skip() {
    # TODO(jmmv): Consider using ${LINENO} if available.
    if [ "${_Shtk_Unittest_ExpectFailure}" = no ]; then
        shtk_cli_warning "${@}"
        touch result.skipped
        exit 0
    else
        shtk_cli_error "Attempted to skip a test while expecting a failure"
    fi
}


# All unittest core functionality defined; let's pull in all checks!
shtk_import unittest/commands
shtk_import unittest/files
shtk_import unittest/operators