aboutsummaryrefslogtreecommitdiffstats
path: root/integration/cmd_report_test.sh
diff options
context:
space:
mode:
Diffstat (limited to 'integration/cmd_report_test.sh')
-rwxr-xr-xintegration/cmd_report_test.sh381
1 files changed, 381 insertions, 0 deletions
diff --git a/integration/cmd_report_test.sh b/integration/cmd_report_test.sh
new file mode 100755
index 000000000000..18a5db386dfd
--- /dev/null
+++ b/integration/cmd_report_test.sh
@@ -0,0 +1,381 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# Executes a mock test suite to generate data in the database.
+#
+# \param mock_env The value to store in a MOCK variable in the environment.
+# Use this to be able to differentiate executions by inspecting the
+# context of the output.
+# \param dbfile_name File to which to write the path to the generated database
+# file.
+run_tests() {
+ local mock_env="${1}"; shift
+ local dbfile_name="${1}"; shift
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="simple_all_pass"}
+EOF
+
+ utils_cp_helper simple_all_pass .
+ atf_check -s exit:0 -o save:stdout -e empty env \
+ MOCK="${mock_env}" _='fake-value' kyua test
+ grep '^Results saved to ' stdout | cut -d ' ' -f 4 >"${dbfile_name}"
+ rm stdout
+
+ # Ensure the results of 'report' come from the database.
+ rm Kyuafile simple_all_pass
+}
+
+
+utils_test_case default_behavior__ok
+default_behavior__ok_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report
+
+ run_tests "mock2" dbfile_name2
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name2)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report
+}
+
+
+utils_test_case default_behavior__no_store
+default_behavior__no_store_body() {
+ echo 'kyua: E: No previous results file found for test suite' \
+ "$(utils_test_suite_id)." >experr
+ atf_check -s exit:2 -o empty -e file:experr kyua report
+}
+
+
+utils_test_case results_file__explicit
+results_file__explicit_body() {
+ run_tests "mock1" dbfile_name1
+ run_tests "mock2" dbfile_name2
+
+ atf_check -s exit:0 -o match:"MOCK=mock1" -o not-match:"MOCK=mock2" \
+ -e empty kyua report --results-file="$(cat dbfile_name1)" \
+ --verbose
+ atf_check -s exit:0 -o not-match:"MOCK=mock1" -o match:"MOCK=mock2" \
+ -e empty kyua report --results-file="$(cat dbfile_name2)" \
+ --verbose
+}
+
+
+utils_test_case results_file__not_found
+results_file__not_found_body() {
+ atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \
+ kyua report --results-file=foo
+}
+
+
+utils_test_case output__explicit
+output__explicit_body() {
+ run_tests unused_mock dbfile_name
+
+ cat >report <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+
+ atf_check -s exit:0 -o file:report -e empty -x kyua report \
+ --output=/dev/stdout "| ${utils_strip_times_but_not_ids}"
+ atf_check -s exit:0 -o empty -e save:stderr kyua report \
+ --output=/dev/stderr
+ atf_check -s exit:0 -o file:report -x cat stderr \
+ "| ${utils_strip_times_but_not_ids}"
+
+ atf_check -s exit:0 -o empty -e empty kyua report \
+ --output=my-file
+ atf_check -s exit:0 -o file:report -x cat my-file \
+ "| ${utils_strip_times_but_not_ids}"
+}
+
+
+utils_test_case filter__ok
+filter__ok_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 1 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ simple_all_pass:skip
+}
+
+
+utils_test_case filter__ok_passed_excluded_by_default
+filter__ok_passed_excluded_by_default_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ # Passed results are excluded by default so they are not displayed even if
+ # requested with a test case filter. This might be somewhat confusing...
+ cat >expout <<EOF
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 1 total, 0 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ simple_all_pass:pass
+ cat >expout <<EOF
+===> Passed tests
+simple_all_pass:pass -> passed [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 1 total, 0 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ --results-filter= simple_all_pass:pass
+}
+
+
+utils_test_case filter__no_match
+filter__no_match_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 1 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ cat >experr <<EOF
+kyua: W: No test cases matched by the filter 'first'.
+kyua: W: No test cases matched by the filter 'simple_all_pass:second'.
+EOF
+ atf_check -s exit:1 -o file:expout -e file:experr kyua report \
+ first simple_all_pass:skip simple_all_pass:second
+}
+
+
+utils_test_case verbose
+verbose_body() {
+ # Switch to the current directory using its physical location and update
+ # HOME accordingly. Otherwise, the test below where we compare the value
+ # of HOME in the output might fail if the path to HOME contains a symlink
+ # (as is the case in OS X when HOME points to the temporary directory.)
+ local real_cwd="$(pwd -P)"
+ cd "${real_cwd}"
+ HOME="${real_cwd}"
+
+ run_tests "mock1
+has multiple lines
+and terminates here" dbfile_name
+
+ cat >expout <<EOF
+===> Execution context
+Current directory: ${real_cwd}
+Environment variables:
+EOF
+ # $_ is a bash variable. To keep our tests stable, we override its value
+ # below to match the hardcoded value in run_tests.
+ env \
+ HOME="${real_cwd}" \
+ MOCK="mock1
+has multiple lines
+and terminates here" \
+ _='fake-value' \
+ "$(atf_get_srcdir)/helpers/dump_env" ' ' ' ' >>expout
+ cat >>expout <<EOF
+===> simple_all_pass:skip
+Result: skipped: The reason for skipping is this
+Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+End time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+Duration: S.UUUs
+
+Metadata:
+ allowed_architectures is empty
+ allowed_platforms is empty
+ description is empty
+ has_cleanup = false
+ is_exclusive = false
+ required_configs is empty
+ required_disk_space = 0
+ required_files is empty
+ required_memory = 0
+ required_programs is empty
+ required_user is empty
+ timeout = 300
+
+Standard output:
+This is the stdout of skip
+
+Standard error:
+This is the stderr of skip
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+End time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty -x kyua report --verbose \
+ "| ${utils_strip_times_but_not_ids}"
+}
+
+
+utils_test_case results_filter__empty
+results_filter__empty_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Passed tests
+simple_all_pass:pass -> passed [S.UUUs]
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report --results-filter=
+}
+
+
+utils_test_case results_filter__one
+results_filter__one_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Passed tests
+simple_all_pass:pass -> passed [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ --results-filter=passed
+}
+
+
+utils_test_case results_filter__multiple_all_match
+results_filter__multiple_all_match_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Passed tests
+simple_all_pass:pass -> passed [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ --results-filter=skipped,passed
+}
+
+
+utils_test_case results_filter__multiple_some_match
+results_filter__multiple_some_match_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ --results-filter=skipped,xfail,broken,failed
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case default_behavior__ok
+ atf_add_test_case default_behavior__no_store
+
+ atf_add_test_case results_file__explicit
+ atf_add_test_case results_file__not_found
+
+ atf_add_test_case filter__ok
+ atf_add_test_case filter__ok_passed_excluded_by_default
+ atf_add_test_case filter__no_match
+
+ atf_add_test_case verbose
+
+ atf_add_test_case output__explicit
+
+ atf_add_test_case results_filter__empty
+ atf_add_test_case results_filter__one
+ atf_add_test_case results_filter__multiple_all_match
+ atf_add_test_case results_filter__multiple_some_match
+}