aboutsummaryrefslogtreecommitdiffstats
path: root/integration
diff options
context:
space:
mode:
authorBrooks Davis <brooks@FreeBSD.org>2020-03-17 16:56:50 +0000
committerBrooks Davis <brooks@FreeBSD.org>2020-03-17 16:56:50 +0000
commit08334c51dbb99d9ecd2bb86a2d94ed06da9e167a (patch)
treec43eb24d59bd5c963583a5190caef80fc8387322 /integration
downloadsrc-vendor/kyua.tar.gz
src-vendor/kyua.zip
Import the kyua testing framework for infrastructure softwarevendor/kyua/0.13-a685f91vendor/kyua
Imported at 0.13 plus assumulated changes to git hash a685f91. Obtained from: https://github.com/jmmv/kyua Sponsored by: DARPA
Notes
Notes: svn path=/vendor/kyua/dist/; revision=359042 svn path=/vendor/kyua/0.13-a685f91/; revision=359043; tag=vendor/kyua/0.13-a685f91
Diffstat (limited to 'integration')
-rw-r--r--integration/Kyuafile16
-rw-r--r--integration/Makefile.am.inc150
-rwxr-xr-xintegration/cmd_about_test.sh158
-rwxr-xr-xintegration/cmd_config_test.sh355
-rwxr-xr-xintegration/cmd_db_exec_test.sh165
-rwxr-xr-xintegration/cmd_db_migrate_test.sh167
-rwxr-xr-xintegration/cmd_debug_test.sh421
-rwxr-xr-xintegration/cmd_help_test.sh93
-rwxr-xr-xintegration/cmd_list_test.sh600
-rwxr-xr-xintegration/cmd_report_html_test.sh267
-rwxr-xr-xintegration/cmd_report_junit_test.sh300
-rwxr-xr-xintegration/cmd_report_test.sh381
-rwxr-xr-xintegration/cmd_test_test.sh1071
-rwxr-xr-xintegration/global_test.sh146
-rw-r--r--integration/helpers/.gitignore11
-rw-r--r--integration/helpers/Makefile.am.inc90
-rw-r--r--integration/helpers/bad_test_program.cpp50
-rw-r--r--integration/helpers/bogus_test_cases.cpp64
-rw-r--r--integration/helpers/config.cpp58
-rw-r--r--integration/helpers/dump_env.cpp74
-rw-r--r--integration/helpers/expect_all_pass.cpp92
-rw-r--r--integration/helpers/expect_some_fail.cpp94
-rw-r--r--integration/helpers/interrupts.cpp62
-rw-r--r--integration/helpers/metadata.cpp95
-rw-r--r--integration/helpers/race.cpp99
-rw-r--r--integration/helpers/simple_all_pass.cpp55
-rw-r--r--integration/helpers/simple_some_fail.cpp53
-rwxr-xr-xintegration/utils.sh177
28 files changed, 5364 insertions, 0 deletions
diff --git a/integration/Kyuafile b/integration/Kyuafile
new file mode 100644
index 000000000000..2ebb4ec8acca
--- /dev/null
+++ b/integration/Kyuafile
@@ -0,0 +1,16 @@
+syntax(2)
+
+test_suite("kyua")
+
+atf_test_program{name="cmd_about_test"}
+atf_test_program{name="cmd_config_test"}
+atf_test_program{name="cmd_db_exec_test"}
+atf_test_program{name="cmd_db_migrate_test"}
+atf_test_program{name="cmd_debug_test"}
+atf_test_program{name="cmd_help_test"}
+atf_test_program{name="cmd_list_test"}
+atf_test_program{name="cmd_report_html_test"}
+atf_test_program{name="cmd_report_junit_test"}
+atf_test_program{name="cmd_report_test"}
+atf_test_program{name="cmd_test_test"}
+atf_test_program{name="global_test"}
diff --git a/integration/Makefile.am.inc b/integration/Makefile.am.inc
new file mode 100644
index 000000000000..cf9ad86d5730
--- /dev/null
+++ b/integration/Makefile.am.inc
@@ -0,0 +1,150 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+if WITH_ATF
+tests_integrationdir = $(pkgtestsdir)/integration
+
+tests_integration_DATA = integration/Kyuafile
+EXTRA_DIST += $(tests_integration_DATA)
+
+ATF_SH_BUILD = \
+ $(MKDIR_P) integration; \
+ echo "\#! $(ATF_SH)" >integration/$${name}; \
+ echo "\# AUTOMATICALLY GENERATED FROM Makefile" >>integration/$${name}; \
+ if [ -n "$${substs}" ]; then \
+ cat $(srcdir)/integration/utils.sh $(srcdir)/integration/$${name}.sh \
+ | sed "$${substs}" >>integration/$${name}; \
+ else \
+ cat $(srcdir)/integration/utils.sh $(srcdir)/integration/$${name}.sh \
+ >>integration/$${name}; \
+ fi; \
+ chmod +x integration/$${name}
+
+ATF_SH_DEPS = \
+ $(srcdir)/integration/utils.sh \
+ Makefile
+
+EXTRA_DIST += integration/utils.sh
+
+tests_integration_SCRIPTS = integration/cmd_about_test
+CLEANFILES += integration/cmd_about_test
+EXTRA_DIST += integration/cmd_about_test.sh
+integration/cmd_about_test: $(srcdir)/integration/cmd_about_test.sh \
+ $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_about_test"; \
+ substs='s,__KYUA_DOCDIR__,$(docdir),g'; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/cmd_config_test
+CLEANFILES += integration/cmd_config_test
+EXTRA_DIST += integration/cmd_config_test.sh
+integration/cmd_config_test: $(srcdir)/integration/cmd_config_test.sh \
+ $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_config_test"; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/cmd_db_exec_test
+CLEANFILES += integration/cmd_db_exec_test
+EXTRA_DIST += integration/cmd_db_exec_test.sh
+integration/cmd_db_exec_test: $(srcdir)/integration/cmd_db_exec_test.sh \
+ $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_db_exec_test"; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/cmd_db_migrate_test
+CLEANFILES += integration/cmd_db_migrate_test
+EXTRA_DIST += integration/cmd_db_migrate_test.sh
+integration/cmd_db_migrate_test: $(srcdir)/integration/cmd_db_migrate_test.sh \
+ $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_db_migrate_test"; \
+ substs='s,__KYUA_STOREDIR__,$(storedir),g'; \
+ substs="$${substs};s,__KYUA_STORETESTDATADIR__,$(tests_storedir),g"; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/cmd_debug_test
+CLEANFILES += integration/cmd_debug_test
+EXTRA_DIST += integration/cmd_debug_test.sh
+integration/cmd_debug_test: $(srcdir)/integration/cmd_debug_test.sh \
+ $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_debug_test"; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/cmd_help_test
+CLEANFILES += integration/cmd_help_test
+EXTRA_DIST += integration/cmd_help_test.sh
+integration/cmd_help_test: $(srcdir)/integration/cmd_help_test.sh $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_help_test"; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/cmd_list_test
+CLEANFILES += integration/cmd_list_test
+EXTRA_DIST += integration/cmd_list_test.sh
+integration/cmd_list_test: $(srcdir)/integration/cmd_list_test.sh $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_list_test"; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/cmd_report_test
+CLEANFILES += integration/cmd_report_test
+EXTRA_DIST += integration/cmd_report_test.sh
+integration/cmd_report_test: $(srcdir)/integration/cmd_report_test.sh \
+ $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_report_test"; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/cmd_report_html_test
+CLEANFILES += integration/cmd_report_html_test
+EXTRA_DIST += integration/cmd_report_html_test.sh
+integration/cmd_report_html_test: \
+ $(srcdir)/integration/cmd_report_html_test.sh $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_report_html_test"; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/cmd_report_junit_test
+CLEANFILES += integration/cmd_report_junit_test
+EXTRA_DIST += integration/cmd_report_junit_test.sh
+integration/cmd_report_junit_test: \
+ $(srcdir)/integration/cmd_report_junit_test.sh $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_report_junit_test"; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/cmd_test_test
+CLEANFILES += integration/cmd_test_test
+EXTRA_DIST += integration/cmd_test_test.sh
+integration/cmd_test_test: $(srcdir)/integration/cmd_test_test.sh $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="cmd_test_test"; \
+ $(ATF_SH_BUILD)
+
+tests_integration_SCRIPTS += integration/global_test
+CLEANFILES += integration/global_test
+EXTRA_DIST += integration/global_test.sh
+integration/global_test: $(srcdir)/integration/global_test.sh $(ATF_SH_DEPS)
+ $(AM_V_GEN)name="global_test"; \
+ $(ATF_SH_BUILD)
+endif
+
+include integration/helpers/Makefile.am.inc
diff --git a/integration/cmd_about_test.sh b/integration/cmd_about_test.sh
new file mode 100755
index 000000000000..06d5da5ac4c2
--- /dev/null
+++ b/integration/cmd_about_test.sh
@@ -0,0 +1,158 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# Location of installed documents. Used to validate the output of the about
+# messages against the golden files.
+: "${KYUA_DOCDIR:=__KYUA_DOCDIR__}"
+
+
+# Common code to validate the output of all about information.
+#
+# \param file The name of the file with the output.
+check_all() {
+ local file="${1}"; shift
+
+ grep -E 'kyua .*[0-9]+\.[0-9]+' "${file}" || \
+ atf_fail 'No version reported'
+ grep 'Copyright' "${file}" || atf_fail 'No license reported'
+ grep '^\*[^<>]*$' "${file}" || atf_fail 'No authors reported'
+ grep '^\*.*<.*@.*>$' "${file}" || atf_fail 'No contributors reported'
+ grep 'Homepage' "${file}" || atf_fail 'No homepage reported'
+}
+
+
+utils_test_case all_topics__installed
+all_topics__installed_head() {
+ atf_set "require.files" "${KYUA_DOCDIR}/AUTHORS" \
+ "${KYUA_DOCDIR}/CONTRIBUTORS" "${KYUA_DOCDIR}/LICENSE"
+}
+all_topics__installed_body() {
+ atf_check -s exit:0 -o save:stdout -e empty kyua about
+ check_all stdout
+}
+
+
+utils_test_case all_topics__override
+all_topics__override_body() {
+ mkdir docs
+ echo "* Author (no email)" >docs/AUTHORS
+ echo "* Contributor <contributor@example.net>" >docs/CONTRIBUTORS
+ echo "Copyright text" >docs/LICENSE
+ export KYUA_DOCDIR=docs
+ atf_check -s exit:0 -o save:stdout -e empty kyua about
+ check_all stdout
+}
+
+
+utils_test_case topic__authors__installed
+topic__authors__installed_head() {
+ atf_set "require.files" "${KYUA_DOCDIR}/AUTHORS" \
+ "${KYUA_DOCDIR}/CONTRIBUTORS"
+}
+topic__authors__installed_body() {
+ grep -h '^\* ' "${KYUA_DOCDIR}/AUTHORS" "${KYUA_DOCDIR}/CONTRIBUTORS" \
+ >expout
+ atf_check -s exit:0 -o file:expout -e empty kyua about authors
+}
+
+
+utils_test_case topic__authors__override
+topic__authors__override_body() {
+ mkdir docs
+ echo "* Author (no email)" >docs/AUTHORS
+ echo "* Contributor <contributor@example.net>" >docs/CONTRIBUTORS
+ export KYUA_DOCDIR=docs
+ cat docs/AUTHORS docs/CONTRIBUTORS >expout
+ atf_check -s exit:0 -o file:expout -e empty kyua about authors
+}
+
+
+utils_test_case topic__license__installed
+topic__license__installed_head() {
+ atf_set "require.files" "${KYUA_DOCDIR}/LICENSE"
+}
+topic__license__installed_body() {
+ atf_check -s exit:0 -o file:"${KYUA_DOCDIR}/LICENSE" -e empty \
+ kyua about license
+}
+
+
+utils_test_case topic__license__override
+topic__license__override_body() {
+ mkdir docs
+ echo "Copyright text" >docs/LICENSE
+ export KYUA_DOCDIR=docs
+ atf_check -s exit:0 -o file:docs/LICENSE -e empty kyua about license
+}
+
+
+utils_test_case topic__version
+topic__version_body() {
+ atf_check -s exit:0 -o save:stdout -e empty kyua about version
+
+ local lines="$(wc -l stdout | awk '{ print $1 }')"
+ [ "${lines}" -eq 1 ] || atf_fail "Version query returned more than one line"
+
+ grep -E '^kyua (.*) [0-9]+\.[0-9]+$' stdout || \
+ atf_fail "Invalid version message"
+}
+
+
+utils_test_case topic__invalid
+topic__invalid_body() {
+ cat >experr <<EOF
+Usage error for command about: Invalid about topic 'foo'.
+Type 'kyua help about' for usage information.
+EOF
+ atf_check -s exit:3 -o empty -e file:experr kyua about foo
+}
+
+
+utils_test_case too_many_arguments
+too_many_arguments_body() {
+ cat >stderr <<EOF
+Usage error for command about: Too many arguments.
+Type 'kyua help about' for usage information.
+EOF
+ atf_check -s exit:3 -o empty -e file:stderr kyua about abc def
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case all_topics__installed
+ atf_add_test_case all_topics__override
+ atf_add_test_case topic__authors__installed
+ atf_add_test_case topic__authors__override
+ atf_add_test_case topic__license__installed
+ atf_add_test_case topic__license__override
+ atf_add_test_case topic__version
+ atf_add_test_case topic__invalid
+
+ atf_add_test_case too_many_arguments
+}
diff --git a/integration/cmd_config_test.sh b/integration/cmd_config_test.sh
new file mode 100755
index 000000000000..ed457e5c4b37
--- /dev/null
+++ b/integration/cmd_config_test.sh
@@ -0,0 +1,355 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+utils_test_case defaults
+defaults_body() {
+ atf_check -s exit:0 \
+ -o match:'^architecture = ' \
+ -o match:'^platform = ' \
+ kyua config
+}
+
+
+utils_test_case all
+all_body() {
+ mkdir "${HOME}/.kyua"
+ cat >"${HOME}/.kyua/kyua.conf" <<EOF
+syntax(2)
+architecture = "my-architecture"
+parallelism = 256
+platform = "my-platform"
+unprivileged_user = "$(id -u -n)"
+test_suites.suite1.the_variable = "value1"
+test_suites.suite2.the_variable = "value2"
+EOF
+
+ cat >expout <<EOF
+architecture = my-architecture
+parallelism = 256
+platform = my-platform
+test_suites.suite1.the_variable = value1
+test_suites.suite2.the_variable = value2
+unprivileged_user = $(id -u -n)
+EOF
+
+ atf_check -s exit:0 -o file:expout -e empty kyua config
+}
+
+
+utils_test_case one__ok
+one__ok_body() {
+ mkdir "${HOME}/.kyua"
+ cat >"${HOME}/.kyua/kyua.conf" <<EOF
+syntax(2)
+test_suites.first.one = 1
+test_suites.first.two = 2
+EOF
+
+ cat >expout <<EOF
+test_suites.first.two = 2
+EOF
+
+ atf_check -s exit:0 -o file:expout -e empty kyua config \
+ test_suites.first.two
+}
+
+
+utils_test_case one__fail
+one__fail_body() {
+ mkdir "${HOME}/.kyua"
+ cat >"${HOME}/.kyua/kyua.conf" <<EOF
+syntax(2)
+test_suites.first.one = 1
+test_suites.first.three = 3
+EOF
+
+ cat >experr <<EOF
+kyua: W: 'test_suites.first.two' is not defined.
+EOF
+
+ atf_check -s exit:1 -o empty -e file:experr kyua config \
+ test_suites.first.two
+}
+
+
+utils_test_case many__ok
+many__ok_body() {
+ mkdir "${HOME}/.kyua"
+ cat >"${HOME}/.kyua/kyua.conf" <<EOF
+syntax(2)
+architecture = "overriden"
+unknown_setting = "foo"
+test_suites.first.one = 1
+test_suites.first.two = 2
+EOF
+
+ cat >expout <<EOF
+architecture = overriden
+test_suites.first.two = 2
+test_suites.first.one = 1
+EOF
+
+ atf_check -s exit:0 -o file:expout -e empty kyua config \
+ architecture \
+ test_suites.first.two \
+ test_suites.first.one # Inverse order on purpose.
+ atf_check -s exit:0 -o match:architecture -o not-match:unknown_setting \
+ -e empty kyua config
+}
+
+
+utils_test_case many__fail
+many__fail_body() {
+ mkdir "${HOME}/.kyua"
+ cat >"${HOME}/.kyua/kyua.conf" <<EOF
+syntax(2)
+test_suites.first.one = 1
+test_suites.first.three = 3
+EOF
+
+ cat >expout <<EOF
+test_suites.first.one = 1
+test_suites.first.three = 3
+EOF
+
+ cat >experr <<EOF
+kyua: W: 'test_suites.first.two' is not defined.
+kyua: W: 'test_suites.first.fourth' is not defined.
+EOF
+
+ atf_check -s exit:1 -o file:expout -e file:experr kyua config \
+ test_suites.first.one test_suites.first.two \
+ test_suites.first.three test_suites.first.fourth
+}
+
+
+utils_test_case config_flag__default_system
+config_flag__default_system_body() {
+ cat >kyua.conf <<EOF
+syntax(2)
+test_suites.foo.var = "baz"
+EOF
+
+ atf_check -s exit:1 -o empty \
+ -e match:"kyua: W: 'test_suites.foo.var'.*not defined" \
+ kyua config test_suites.foo.var
+ export KYUA_CONFDIR="$(pwd)"
+ atf_check -s exit:0 -o match:"foo.var = baz" -e empty \
+ kyua config test_suites.foo.var
+}
+
+
+utils_test_case config_flag__default_home
+config_flag__default_home_body() {
+ cat >kyua.conf <<EOF
+syntax(2)
+test_suites.foo.var = "bar"
+EOF
+ export KYUA_CONFDIR="$(pwd)"
+ atf_check -s exit:0 -o match:"test_suites.foo.var = bar" -e empty \
+ kyua config test_suites.foo.var
+
+ # The previously-created "system-wide" file has to be ignored.
+ mkdir .kyua
+ cat >.kyua/kyua.conf <<EOF
+syntax(2)
+test_suites.foo.var = "baz"
+EOF
+ atf_check -s exit:0 -o match:"test_suites.foo.var = baz" -e empty \
+ kyua config test_suites.foo.var
+}
+
+
+utils_test_case config_flag__explicit__ok
+config_flag__explicit__ok_body() {
+ cat >kyua.conf <<EOF
+syntax(2)
+test_suites.foo.var = "baz"
+EOF
+
+ atf_check -s exit:1 -o empty \
+ -e match:"kyua: W: 'test_suites.foo.var'.*not defined" \
+ kyua config test_suites.foo.var
+ atf_check -s exit:0 -o match:"test_suites.foo.var = baz" -e empty \
+ kyua -c kyua.conf config test_suites.foo.var
+ atf_check -s exit:0 -o match:"test_suites.foo.var = baz" -e empty \
+ kyua --config=kyua.conf config test_suites.foo.var
+}
+
+
+utils_test_case config_flag__explicit__disable
+config_flag__explicit__disable_body() {
+ cat >kyua.conf <<EOF
+syntax(2)
+test_suites.foo.var = "baz"
+EOF
+ mkdir .kyua
+ cp kyua.conf .kyua/kyua.conf
+ export KYUA_CONFDIR="$(pwd)"
+
+ atf_check -s exit:0 -o match:"test_suites.foo.var = baz" -e empty \
+ kyua config test_suites.foo.var
+ atf_check -s exit:1 -o empty \
+ -e match:"kyua: W: 'test_suites.foo.var'.*not defined" \
+ kyua --config=none config test_suites.foo.var
+}
+
+
+utils_test_case config_flag__explicit__missing_file
+config_flag__explicit__missing_file_body() {
+ cat >experr <<EOF
+kyua: E: Load of 'foo' failed: File 'foo' not found.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua --config=foo config
+}
+
+
+utils_test_case config_flag__explicit__bad_file
+config_flag__explicit__bad_file_body() {
+ touch custom
+ atf_check -s exit:2 -o empty -e match:"No syntax defined" \
+ kyua --config=custom config
+}
+
+
+utils_test_case variable_flag__no_config
+variable_flag__no_config_body() {
+ atf_check -s exit:0 \
+ -o match:'test_suites.suite1.the_variable = value1' \
+ -o match:'test_suites.suite2.the_variable = value2' \
+ -e empty \
+ kyua \
+ -v "test_suites.suite1.the_variable=value1" \
+ -v "test_suites.suite2.the_variable=value2" \
+ config
+
+ atf_check -s exit:0 \
+ -o match:'test_suites.suite1.the_variable = value1' \
+ -o match:'test_suites.suite2.the_variable = value2' \
+ -e empty \
+ kyua \
+ --variable="test_suites.suite1.the_variable=value1" \
+ --variable="test_suites.suite2.the_variable=value2" \
+ config
+}
+
+
+utils_test_case variable_flag__override_default_config
+variable_flag__override_default_config_body() {
+ mkdir "${HOME}/.kyua"
+ cat >"${HOME}/.kyua/kyua.conf" <<EOF
+syntax(2)
+test_suites.suite1.the_variable = "value1"
+test_suites.suite2.the_variable = "should not be used"
+EOF
+
+ atf_check -s exit:0 \
+ -o match:'test_suites.suite1.the_variable = value1' \
+ -o match:'test_suites.suite2.the_variable = overriden' \
+ -o match:'test_suites.suite3.the_variable = new' \
+ -e empty kyua \
+ -v "test_suites.suite2.the_variable=overriden" \
+ -v "test_suites.suite3.the_variable=new" \
+ config
+
+ atf_check -s exit:0 \
+ -o match:'test_suites.suite1.the_variable = value1' \
+ -o match:'test_suites.suite2.the_variable = overriden' \
+ -o match:'test_suites.suite3.the_variable = new' \
+ -e empty kyua \
+ --variable="test_suites.suite2.the_variable=overriden" \
+ --variable="test_suites.suite3.the_variable=new" \
+ config
+}
+
+
+utils_test_case variable_flag__override_custom_config
+variable_flag__override_custom_config_body() {
+ cat >config <<EOF
+syntax(2)
+test_suites.suite1.the_variable = "value1"
+test_suites.suite2.the_variable = "should not be used"
+EOF
+
+ atf_check -s exit:0 \
+ -o match:'test_suites.suite2.the_variable = overriden' \
+ -e empty kyua -c config \
+ -v "test_suites.suite2.the_variable=overriden" config
+
+ atf_check -s exit:0 \
+ -o match:'test_suites.suite2.the_variable = overriden' \
+ -e empty kyua -c config \
+ --variable="test_suites.suite2.the_variable=overriden" config
+}
+
+
+utils_test_case variable_flag__invalid_key
+variable_flag__invalid_key_body() {
+ # CHECK_STYLE_DISABLE
+ cat >experr <<EOF
+Usage error: Invalid argument '' for option --variable: Argument does not have the form 'K=V'.
+Type 'kyua help' for usage information.
+EOF
+ # CHECK_STYLE_ENABLE
+ atf_check -s exit:3 -o empty -e file:experr kyua \
+ -v "test_suites.a.b=c" -v "" config
+}
+
+
+utils_test_case variable_flag__invalid_value
+variable_flag__invalid_value_body() {
+ cat >experr <<EOF
+kyua: E: Invalid value for property 'parallelism': Must be a positive integer.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua \
+ -v "parallelism=0" config
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case defaults
+ atf_add_test_case all
+ atf_add_test_case one__ok
+ atf_add_test_case one__fail
+ atf_add_test_case many__ok
+ atf_add_test_case many__fail
+
+ atf_add_test_case config_flag__default_system
+ atf_add_test_case config_flag__default_home
+ atf_add_test_case config_flag__explicit__ok
+ atf_add_test_case config_flag__explicit__disable
+ atf_add_test_case config_flag__explicit__missing_file
+ atf_add_test_case config_flag__explicit__bad_file
+
+ atf_add_test_case variable_flag__no_config
+ atf_add_test_case variable_flag__override_default_config
+ atf_add_test_case variable_flag__override_custom_config
+ atf_add_test_case variable_flag__invalid_key
+ atf_add_test_case variable_flag__invalid_value
+}
diff --git a/integration/cmd_db_exec_test.sh b/integration/cmd_db_exec_test.sh
new file mode 100755
index 000000000000..c260f23a78e1
--- /dev/null
+++ b/integration/cmd_db_exec_test.sh
@@ -0,0 +1,165 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# Creates a new database file in the store directory.
+#
+# Subsequent invocations of db-exec should just pick this new file up.
+create_empty_store() {
+ cat >Kyuafile <<EOF
+syntax(2)
+EOF
+ atf_check -s exit:0 -o ignore -e empty kyua test
+}
+
+
+utils_test_case one_arg
+one_arg_body() {
+ create_empty_store
+
+ atf_check -s exit:0 -o save:metadata.csv -e empty \
+ kyua db-exec "SELECT * FROM metadata"
+ atf_check -s exit:0 -o ignore -e empty \
+ grep 'schema_version,.*timestamp' metadata.csv
+}
+
+
+utils_test_case many_args
+many_args_body() {
+ create_empty_store
+
+ atf_check -s exit:0 -o save:metadata.csv -e empty \
+ kyua db-exec SELECT "*" FROM metadata
+ atf_check -s exit:0 -o ignore -e empty \
+ grep 'schema_version,.*timestamp' metadata.csv
+}
+
+
+utils_test_case no_args
+no_args_body() {
+ atf_check -s exit:3 -o empty -e match:"Not enough arguments" kyua db-exec
+ test ! -d .kyua/store/ || atf_fail "Database created but it should" \
+ "not have been"
+}
+
+
+utils_test_case invalid_statement
+invalid_statement_body() {
+ create_empty_store
+
+ atf_check -s exit:1 -o empty -e match:"SQLite error.*foo" \
+ kyua db-exec foo
+}
+
+
+utils_test_case no_create_store
+no_create_store_body() {
+ atf_check -s exit:1 -o empty -e match:"No previous results.*not-here" \
+ kyua db-exec --results-file=not-here "SELECT * FROM metadata"
+ if [ -f not-here ]; then
+ atf_fail "Database created but it should not have been"
+ fi
+}
+
+
+utils_test_case results_file__default_home
+results_file__default_home_body() {
+ HOME=home-dir
+ create_empty_store
+
+ atf_check -s exit:0 -o save:metadata.csv -e empty \
+ kyua db-exec "SELECT * FROM metadata"
+ test -f home-dir/.kyua/store/*.db || atf_fail "Database not created in" \
+ "the home directory"
+ atf_check -s exit:0 -o ignore -e empty \
+ grep 'schema_version,.*timestamp' metadata.csv
+}
+
+
+utils_test_case results_file__explicit__ok
+results_file__explicit__ok_body() {
+ create_empty_store
+ mv .kyua/store/*.db custom.db
+ rmdir .kyua/store
+
+ HOME=home-dir
+ atf_check -s exit:0 -o save:metadata.csv -e empty \
+ kyua --logfile=/dev/null db-exec -r custom.db "SELECT * FROM metadata"
+ test ! -d home-dir/.kyua || atf_fail "Home directory created but this" \
+ "should not have happened"
+ atf_check -s exit:0 -o ignore -e empty \
+ grep 'schema_version,.*timestamp' metadata.csv
+}
+
+
+utils_test_case results_file__explicit__fail
+results_file__explicit__fail_head() {
+ atf_set "require.user" "unprivileged"
+}
+results_file__explicit__fail_body() {
+ atf_check -s exit:1 -o empty -e match:"No previous results.*foo.db" \
+ kyua db-exec --results-file=foo.db "SELECT * FROM metadata"
+}
+
+
+utils_test_case no_headers_flag
+no_headers_flag_body() {
+ create_empty_store
+
+ atf_check kyua db-exec "CREATE TABLE data" \
+ "(a INTEGER PRIMARY KEY, b INTEGER, c TEXT)"
+ atf_check kyua db-exec "INSERT INTO data VALUES (65, 43, NULL)"
+ atf_check kyua db-exec "INSERT INTO data VALUES (23, 42, 'foo')"
+
+ cat >expout <<EOF
+a,b,c
+23,42,foo
+65,43,NULL
+EOF
+ atf_check -s exit:0 -o file:expout -e empty \
+ kyua db-exec "SELECT * FROM data ORDER BY a"
+
+ tail -n 2 <expout >expout2
+ atf_check -s exit:0 -o file:expout2 -e empty \
+ kyua db-exec --no-headers "SELECT * FROM data ORDER BY a"
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case one_arg
+ atf_add_test_case many_args
+ atf_add_test_case no_args
+ atf_add_test_case invalid_statement
+ atf_add_test_case no_create_store
+
+ atf_add_test_case results_file__default_home
+ atf_add_test_case results_file__explicit__ok
+ atf_add_test_case results_file__explicit__fail
+
+ atf_add_test_case no_headers_flag
+}
diff --git a/integration/cmd_db_migrate_test.sh b/integration/cmd_db_migrate_test.sh
new file mode 100755
index 000000000000..404a4e774019
--- /dev/null
+++ b/integration/cmd_db_migrate_test.sh
@@ -0,0 +1,167 @@
+# Copyright 2013 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# Location of installed schema files.
+: "${KYUA_STOREDIR:=__KYUA_STOREDIR__}"
+
+
+# Location of installed test data files.
+: "${KYUA_STORETESTDATADIR:=__KYUA_STORETESTDATADIR__}"
+
+
+# Creates an empty old-style action database.
+#
+# \param ... Files that contain SQL commands to be run.
+create_historical_db() {
+ mkdir -p "${HOME}/.kyua"
+ cat "${@}" | sqlite3 "${HOME}/.kyua/store.db"
+}
+
+
+# Creates an empty results file.
+#
+# \param ... Files that contain SQL commands to be run.
+create_results_file() {
+ mkdir -p "${HOME}/.kyua/store"
+ local dbname="results.$(utils_test_suite_id)-20140718-173200-123456.db"
+ cat "${@}" | sqlite3 "${HOME}/.kyua/store/${dbname}"
+}
+
+
+utils_test_case upgrade__from_v1
+upgrade__from_v1_head() {
+ atf_set require.files \
+ "${KYUA_STORETESTDATADIR}/schema_v1.sql" \
+ "${KYUA_STORETESTDATADIR}/testdata_v1.sql" \
+ "${KYUA_STOREDIR}/migrate_v1_v2.sql" \
+ "${KYUA_STOREDIR}/migrate_v2_v3.sql"
+ atf_set require.progs "sqlite3"
+}
+upgrade__from_v1_body() {
+ create_historical_db "${KYUA_STORETESTDATADIR}/schema_v1.sql" \
+ "${KYUA_STORETESTDATADIR}/testdata_v1.sql"
+ atf_check -s exit:0 -o empty -e empty kyua db-migrate
+ for f in \
+ "results.test_suite_root.20130108-111331-000000.db" \
+ "results.usr_tests.20130108-123832-000000.db" \
+ "results.usr_tests.20130108-112635-000000.db"
+ do
+ [ -f "${HOME}/.kyua/store/${f}" ] || atf_fail "Expected file ${f}" \
+ "was not created"
+ done
+ [ ! -f "${HOME}/.kyua/store.db" ] || atf_fail "Historical database not" \
+ "deleted"
+}
+
+
+utils_test_case upgrade__from_v2
+upgrade__from_v2_head() {
+ atf_set require.files \
+ "${KYUA_STORETESTDATADIR}/schema_v2.sql" \
+ "${KYUA_STORETESTDATADIR}/testdata_v2.sql" \
+ "${KYUA_STOREDIR}/migrate_v2_v3.sql"
+ atf_set require.progs "sqlite3"
+}
+upgrade__from_v2_body() {
+ create_historical_db "${KYUA_STORETESTDATADIR}/schema_v2.sql" \
+ "${KYUA_STORETESTDATADIR}/testdata_v2.sql"
+ atf_check -s exit:0 -o empty -e empty kyua db-migrate
+ for f in \
+ "results.test_suite_root.20130108-111331-000000.db" \
+ "results.usr_tests.20130108-123832-000000.db" \
+ "results.usr_tests.20130108-112635-000000.db"
+ do
+ [ -f "${HOME}/.kyua/store/${f}" ] || atf_fail "Expected file ${f}" \
+ "was not created"
+ done
+ [ ! -f "${HOME}/.kyua/store.db" ] || atf_fail "Historical database not" \
+ "deleted"
+}
+
+
+utils_test_case already_up_to_date
+already_up_to_date_head() {
+ atf_set require.files "${KYUA_STOREDIR}/schema_v3.sql"
+ atf_set require.progs "sqlite3"
+}
+already_up_to_date_body() {
+ create_results_file "${KYUA_STOREDIR}/schema_v3.sql"
+ atf_check -s exit:1 -o empty -e match:"already at schema version" \
+ kyua db-migrate
+}
+
+
+utils_test_case need_upgrade
+need_upgrade_head() {
+ atf_set require.files "${KYUA_STORETESTDATADIR}/schema_v1.sql"
+ atf_set require.progs "sqlite3"
+}
+need_upgrade_body() {
+ create_results_file "${KYUA_STORETESTDATADIR}/schema_v1.sql"
+ atf_check -s exit:2 -o empty \
+ -e match:"database has schema version 1.*use db-migrate" kyua report
+}
+
+
+utils_test_case results_file__ok
+results_file__ok_body() {
+ echo "This is not a valid database" >test.db
+ atf_check -s exit:1 -o empty -e match:"Migration failed" \
+ kyua db-migrate --results-file ./test.db
+}
+
+
+utils_test_case results_file__fail
+results_file__fail_body() {
+ atf_check -s exit:1 -o empty -e match:"No previous results.*test.db" \
+ kyua db-migrate --results-file ./test.db
+}
+
+
+utils_test_case too_many_arguments
+too_many_arguments_body() {
+ cat >stderr <<EOF
+Usage error for command db-migrate: Too many arguments.
+Type 'kyua help db-migrate' for usage information.
+EOF
+ atf_check -s exit:3 -o empty -e file:stderr kyua db-migrate abc def
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case upgrade__from_v1
+ atf_add_test_case upgrade__from_v2
+ atf_add_test_case already_up_to_date
+ atf_add_test_case need_upgrade
+
+ atf_add_test_case results_file__ok
+ atf_add_test_case results_file__fail
+
+ atf_add_test_case too_many_arguments
+}
diff --git a/integration/cmd_debug_test.sh b/integration/cmd_debug_test.sh
new file mode 100755
index 000000000000..b34a96d72eda
--- /dev/null
+++ b/integration/cmd_debug_test.sh
@@ -0,0 +1,421 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+utils_test_case no_args
+no_args_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="simple_all_pass"}
+EOF
+ utils_cp_helper simple_all_pass .
+
+ cat >experr <<EOF
+Usage error for command debug: Not enough arguments.
+Type 'kyua help debug' for usage information.
+EOF
+ atf_check -s exit:3 -o empty -e file:experr kyua debug
+}
+
+
+utils_test_case many_args
+many_args_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_all_pass second
+
+ cat >experr <<EOF
+Usage error for command debug: Too many arguments.
+Type 'kyua help debug' for usage information.
+EOF
+ atf_check -s exit:3 -o empty -e file:experr kyua debug first:pass \
+ second:pass
+}
+
+
+utils_test_case one_arg__ok_pass
+one_arg__ok_pass_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper expect_all_pass first
+ utils_cp_helper simple_all_pass second
+
+ cat >expout <<EOF
+This is the stdout of pass
+second:pass -> passed
+EOF
+cat >experr <<EOF
+This is the stderr of pass
+EOF
+ atf_check -s exit:0 -o file:expout -e file:experr kyua debug second:pass
+}
+
+
+utils_test_case one_arg__ok_fail
+one_arg__ok_fail_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+EOF
+ utils_cp_helper simple_some_fail first
+
+ cat >expout <<EOF
+This is the stdout of fail
+first:fail -> failed: This fails on purpose
+EOF
+ cat >experr <<EOF
+This is the stderr of fail
+EOF
+ atf_check -s exit:1 -o file:expout -e file:experr kyua debug first:fail
+}
+
+
+utils_test_case one_arg__no_match
+one_arg__no_match_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper expect_all_pass first
+ utils_cp_helper simple_all_pass second
+
+ cat >experr <<EOF
+kyua: E: Unknown test case 'second:die'.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua debug second:die
+}
+
+
+utils_test_case one_arg__no_test_case
+one_arg__no_test_case_body() {
+ # CHECK_STYLE_DISABLE
+ cat >experr <<EOF
+Usage error for command debug: 'foo' is not a test case identifier (missing ':'?).
+Type 'kyua help debug' for usage information.
+EOF
+ # CHECK_STYLE_ENABLE
+ atf_check -s exit:3 -o empty -e file:experr kyua debug foo
+}
+
+
+utils_test_case one_arg__bad_filter
+one_arg__bad_filter_body() {
+ cat >experr <<EOF
+kyua: E: Test case component in 'foo:' is empty.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua debug foo:
+}
+
+
+utils_test_case body_and_cleanup
+body_and_cleanup_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="single"}
+EOF
+ utils_cp_helper metadata single
+
+ cat >expout <<EOF
+single:with_cleanup -> passed
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua debug \
+ --stdout=saved.out --stderr=saved.err single:with_cleanup
+
+ cat >expout <<EOF
+Body message to stdout
+Cleanup message to stdout
+EOF
+ atf_check -s exit:0 -o file:expout -e empty cat saved.out
+
+ cat >experr <<EOF
+Body message to stderr
+Cleanup message to stderr
+EOF
+ atf_check -s exit:0 -o file:experr -e empty cat saved.err
+}
+
+
+utils_test_case stdout_stderr_flags
+stdout_stderr_flags_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper expect_all_pass first
+ utils_cp_helper simple_all_pass second
+
+ cat >expout <<EOF
+second:pass -> passed
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua debug \
+ --stdout=saved.out --stderr=saved.err second:pass
+
+ cat >expout <<EOF
+This is the stdout of pass
+EOF
+ cmp -s saved.out expout || atf_fail "--stdout did not redirect the" \
+ "standard output to the desired file"
+
+ cat >experr <<EOF
+This is the stderr of pass
+EOF
+ cmp -s saved.err experr || atf_fail "--stderr did not redirect the" \
+ "standard error to the desired file"
+}
+
+
+utils_test_case args_are_relative
+args_are_relative_body() {
+ mkdir root
+ cat >root/Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+include("subdir/Kyuafile")
+atf_test_program{name="prog"}
+EOF
+ utils_cp_helper simple_all_pass root/prog
+
+ mkdir root/subdir
+ cat >root/subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="prog"}
+EOF
+ utils_cp_helper simple_some_fail root/subdir/prog
+
+ cat >expout <<EOF
+This is the stdout of fail
+subdir/prog:fail -> failed: This fails on purpose
+EOF
+ cat >experr <<EOF
+This is the stderr of fail
+EOF
+ atf_check -s exit:1 -o file:expout -e file:experr kyua debug \
+ -k "$(pwd)/root/Kyuafile" subdir/prog:fail
+}
+
+
+utils_test_case only_load_used_test_programs
+only_load_used_test_programs_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper bad_test_program second
+
+ cat >expout <<EOF
+This is the stdout of pass
+first:pass -> passed
+EOF
+ cat >experr <<EOF
+This is the stderr of pass
+EOF
+ CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE
+ atf_check -s exit:0 -o file:expout -e file:experr kyua debug first:pass
+ if [ -f "${CREATE_COOKIE}" ]; then
+ atf_fail "An unmatched test case has been executed, which harms" \
+ "performance"
+ fi
+}
+
+
+utils_test_case config_behavior
+config_behavior_body() {
+ cat >"my-config" <<EOF
+syntax(2)
+test_suites.suite1["the-variable"] = "value1"
+test_suites.suite2["the-variable"] = "override me"
+EOF
+
+ cat >Kyuafile <<EOF
+syntax(2)
+atf_test_program{name="config1", test_suite="suite1"}
+atf_test_program{name="config2", test_suite="suite2"}
+atf_test_program{name="config3", test_suite="suite3"}
+EOF
+ utils_cp_helper config config1
+ utils_cp_helper config config2
+ utils_cp_helper config config3
+
+ atf_check -s exit:1 -o match:'failed' -e empty \
+ kyua -c my-config -v test_suites.suite2.the-variable=value2 \
+ debug config1:get_variable
+ atf_check -s exit:0 -o match:'passed' -e empty \
+ kyua -c my-config -v test_suites.suite2.the-variable=value2 \
+ debug config2:get_variable
+ atf_check -s exit:0 -o match:'skipped' -e empty \
+ kyua -c my-config -v test_suites.suite2.the-variable=value2 \
+ debug config3:get_variable
+}
+
+
+utils_test_case build_root_flag
+build_root_flag_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ mkdir build
+ utils_cp_helper expect_all_pass build/first
+ utils_cp_helper simple_all_pass build/second
+
+ cat >expout <<EOF
+This is the stdout of pass
+second:pass -> passed
+EOF
+cat >experr <<EOF
+This is the stderr of pass
+EOF
+ atf_check -s exit:0 -o file:expout -e file:experr \
+ kyua debug --build-root=build second:pass
+}
+
+
+utils_test_case kyuafile_flag__ok
+kyuafile_flag__ok_body() {
+ cat >Kyuafile <<EOF
+This file is bogus but it is not loaded.
+EOF
+
+ cat >myfile <<EOF
+syntax(2)
+test_suite("hello-world")
+atf_test_program{name="sometest"}
+EOF
+ utils_cp_helper simple_all_pass sometest
+
+ atf_check -s exit:0 -o match:passed -e empty kyua test -k myfile sometest
+ atf_check -s exit:0 -o match:passed -e empty kyua test --kyuafile=myfile \
+ sometest
+}
+
+
+utils_test_case missing_kyuafile
+missing_kyuafile_body() {
+ cat >experr <<EOF
+kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua debug foo:bar
+}
+
+
+utils_test_case bogus_kyuafile
+bogus_kyuafile_body() {
+ cat >Kyuafile <<EOF
+Hello, world.
+EOF
+ atf_check -s exit:2 -o empty \
+ -e match:"Load of 'Kyuafile' failed: .* Kyuafile:2:" kyua list
+}
+
+
+utils_test_case bogus_test_program
+bogus_test_program_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="crash_on_list"}
+atf_test_program{name="non_executable"}
+EOF
+ utils_cp_helper bad_test_program crash_on_list
+ echo 'I am not executable' >non_executable
+
+ cat >experr <<EOF
+kyua: E: Unknown test case 'crash_on_list:a'.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua debug crash_on_list:a
+
+ cat >experr <<EOF
+kyua: E: Unknown test case 'non_executable:a'.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua debug non_executable:a
+
+ # CHECK_STYLE_DISABLE
+ cat >expout <<EOF
+crash_on_list:__test_cases_list__ -> broken: Invalid header for test case list; expecting Content-Type for application/X-atf-tp version 1, got ''
+EOF
+ # CHECK_STYLE_ENABLE
+ atf_check -s exit:1 -o file:expout -e empty kyua debug \
+ crash_on_list:__test_cases_list__
+
+ # CHECK_STYLE_DISABLE
+ cat >expout <<EOF
+non_executable:__test_cases_list__ -> broken: Permission denied to run test program
+EOF
+ # CHECK_STYLE_ENABLE
+ atf_check -s exit:1 -o file:expout -e empty kyua debug \
+ non_executable:__test_cases_list__
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case no_args
+ atf_add_test_case many_args
+ atf_add_test_case one_arg__ok_pass
+ atf_add_test_case one_arg__ok_fail
+ atf_add_test_case one_arg__no_match
+ atf_add_test_case one_arg__no_test_case
+ atf_add_test_case one_arg__bad_filter
+
+ atf_add_test_case body_and_cleanup
+
+ atf_add_test_case stdout_stderr_flags
+
+ atf_add_test_case args_are_relative
+
+ atf_add_test_case only_load_used_test_programs
+
+ atf_add_test_case config_behavior
+
+ atf_add_test_case build_root_flag
+ atf_add_test_case kyuafile_flag__ok
+ atf_add_test_case missing_kyuafile
+ atf_add_test_case bogus_kyuafile
+ atf_add_test_case bogus_test_program
+}
diff --git a/integration/cmd_help_test.sh b/integration/cmd_help_test.sh
new file mode 100755
index 000000000000..d8afbd0e6aba
--- /dev/null
+++ b/integration/cmd_help_test.sh
@@ -0,0 +1,93 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+utils_test_case global
+global_body() {
+ atf_check -s exit:0 -o save:stdout -e empty kyua help
+ grep -E 'kyua .*[0-9]+\.[0-9]+' stdout || atf_fail 'No version reported'
+ grep '^Usage: kyua' stdout || atf_fail 'No usage line printed'
+ grep -- '--loglevel' stdout || atf_fail 'Generic options not printed'
+ if grep -- '--show' stdout; then
+ atf_fail 'One option of the about subcommand appeared in the output'
+ fi
+ grep 'about *Shows detailed' stdout || atf_fail 'Commands not printed'
+}
+
+
+utils_test_case one_command
+one_command_body() {
+ atf_check -s exit:0 -o save:stdout -e empty kyua help test
+ grep -E 'kyua .*[0-9]+\.[0-9]+' stdout || atf_fail 'No version reported'
+ grep '^Usage: kyua' stdout || atf_fail 'No usage line printed'
+ grep '^Run tests' stdout || atf_fail 'No description printed'
+ grep -- '--loglevel' stdout || atf_fail 'Generic options not printed'
+ grep -- '--kyuafile' stdout || atf_fail 'Command options not printed'
+ if grep 'about: Shows detailed' stdout; then
+ atf_fail 'Printed table of commands, but should not have done so'
+ fi
+}
+
+
+utils_test_case ignore_bad_config
+ignore_bad_config_body() {
+ echo 'this is an invalid configuration file' >bad-config
+ atf_check -s exit:0 -o save:stdout -e empty kyua -c bad-config help
+ grep '^Usage: kyua' stdout || atf_fail 'No usage line printed'
+ grep -- '--loglevel' stdout || atf_fail 'Generic options not printed'
+}
+
+
+utils_test_case unknown_command
+unknown_command_body() {
+ cat >stderr <<EOF
+Usage error for command help: The command abc does not exist.
+Type 'kyua help help' for usage information.
+EOF
+ atf_check -s exit:3 -o empty -e file:stderr kyua help abc
+}
+
+
+utils_test_case too_many_arguments
+too_many_arguments_body() {
+ cat >stderr <<EOF
+Usage error for command help: Too many arguments.
+Type 'kyua help help' for usage information.
+EOF
+ atf_check -s exit:3 -o empty -e file:stderr kyua help about cde
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case global
+ atf_add_test_case one_command
+
+ atf_add_test_case ignore_bad_config
+ atf_add_test_case unknown_command
+ atf_add_test_case too_many_arguments
+}
diff --git a/integration/cmd_list_test.sh b/integration/cmd_list_test.sh
new file mode 100755
index 000000000000..a916e0f2ec4b
--- /dev/null
+++ b/integration/cmd_list_test.sh
@@ -0,0 +1,600 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+utils_test_case no_args
+no_args_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="metadata"}
+atf_test_program{name="simple_all_pass"}
+include("subdir/Kyuafile")
+EOF
+ utils_cp_helper metadata .
+ utils_cp_helper simple_all_pass .
+
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration2")
+atf_test_program{name="simple_some_fail"}
+EOF
+ utils_cp_helper simple_some_fail subdir
+
+ cat >expout <<EOF
+metadata:many_properties
+metadata:no_properties
+metadata:one_property
+metadata:with_cleanup
+simple_all_pass:pass
+simple_all_pass:skip
+subdir/simple_some_fail:fail
+subdir/simple_some_fail:pass
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list
+}
+
+
+utils_test_case one_arg__subdir
+one_arg__subdir_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+include("subdir/Kyuafile")
+EOF
+
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("in-subdir")
+atf_test_program{name="simple_all_pass"}
+EOF
+ utils_cp_helper simple_all_pass subdir
+
+ cat >expout <<EOF
+subdir/simple_all_pass:pass
+subdir/simple_all_pass:skip
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list subdir
+}
+
+
+utils_test_case one_arg__test_case
+one_arg__test_case_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_all_pass second
+
+ cat >expout <<EOF
+first:skip
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list first:skip
+}
+
+
+utils_test_case one_arg__test_program
+one_arg__test_program_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_some_fail second
+
+ cat >expout <<EOF
+second:fail
+second:pass
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list second
+}
+
+
+utils_test_case one_arg__invalid
+one_arg__invalid_body() {
+cat >experr <<EOF
+kyua: E: Test case component in 'foo:' is empty.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua list foo:
+
+cat >experr <<EOF
+kyua: E: Program name '/a/b' must be relative to the test suite, not absolute.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua list /a/b
+}
+
+
+utils_test_case many_args__ok
+many_args__ok_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+include("subdir/Kyuafile")
+atf_test_program{name="first"}
+EOF
+ utils_cp_helper simple_all_pass first
+
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("in-subdir")
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_some_fail subdir/second
+
+ cat >expout <<EOF
+subdir/second:fail (in-subdir)
+subdir/second:pass (in-subdir)
+first:pass (top-level)
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list -v subdir first:pass
+}
+
+
+utils_test_case many_args__invalid
+many_args__invalid_body() {
+cat >experr <<EOF
+kyua: E: Program name component in ':badbad' is empty.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua list this-is-ok :badbad
+
+cat >experr <<EOF
+kyua: E: Program name '/foo' must be relative to the test suite, not absolute.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua list this-is-ok /foo
+}
+
+
+utils_test_case many_args__no_match__all
+many_args__no_match__all_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_all_pass second
+
+ cat >experr <<EOF
+kyua: W: No test cases matched by the filter 'first1'.
+EOF
+ atf_check -s exit:1 -o empty -e file:experr kyua list first1
+}
+
+
+utils_test_case many_args__no_match__some
+many_args__no_match__some_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+atf_test_program{name="third"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_all_pass second
+ utils_cp_helper simple_some_fail third
+
+ cat >expout <<EOF
+first:pass
+first:skip
+third:fail
+third:pass
+EOF
+
+ cat >experr <<EOF
+kyua: W: No test cases matched by the filter 'fifth'.
+kyua: W: No test cases matched by the filter 'fourth'.
+EOF
+ atf_check -s exit:1 -o file:expout -e file:experr kyua list first fourth \
+ third fifth
+}
+
+
+utils_test_case args_are_relative
+args_are_relative_body() {
+ mkdir root
+ cat >root/Kyuafile <<EOF
+syntax(2)
+test_suite("integration-1")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+include("subdir/Kyuafile")
+EOF
+ utils_cp_helper simple_all_pass root/first
+ utils_cp_helper simple_some_fail root/second
+
+ mkdir root/subdir
+ cat >root/subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration-2")
+atf_test_program{name="third"}
+atf_test_program{name="fourth"}
+EOF
+ utils_cp_helper simple_all_pass root/subdir/third
+ utils_cp_helper simple_some_fail root/subdir/fourth
+
+ cat >expout <<EOF
+first:pass (integration-1)
+first:skip (integration-1)
+subdir/fourth:fail (integration-2)
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list \
+ -v -k "$(pwd)/root/Kyuafile" first subdir/fourth:fail
+}
+
+
+utils_test_case only_load_used_test_programs
+only_load_used_test_programs_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper bad_test_program second
+
+ cat >expout <<EOF
+first:pass
+first:skip
+EOF
+ CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE
+ atf_check -s exit:0 -o file:expout -e empty kyua list first
+ if [ -f "${CREATE_COOKIE}" ]; then
+ atf_fail "An unmatched test case has been executed, which harms" \
+ "performance"
+ fi
+}
+
+
+utils_test_case config_behavior
+config_behavior_body() {
+ cat >"my-config" <<EOF
+syntax(2)
+test_suites.suite1["the-variable"] = "value1"
+EOF
+
+ cat >Kyuafile <<EOF
+syntax(2)
+atf_test_program{name="config1", test_suite="suite1"}
+EOF
+ utils_cp_helper config config1
+
+ CONFIG_VAR_FILE="$(pwd)/cookie"; export CONFIG_VAR_FILE
+ if [ -f "${CONFIG_VAR_FILE}" ]; then
+ atf_fail "Cookie file already created; test case list may have gotten" \
+ "a bad configuration"
+ fi
+ atf_check -s exit:0 -o ignore -e empty kyua -c my-config list
+ [ -f "${CONFIG_VAR_FILE}" ] || \
+ atf_fail "Cookie file not created; test case list did not get" \
+ "configuration variables"
+ value="$(cat "${CONFIG_VAR_FILE}")"
+ [ "${value}" = "value1" ] || \
+ atf_fail "Invalid value (${value}) in cookie file; test case list did" \
+ "not get the correct configuration variables"
+}
+
+
+utils_test_case build_root_flag
+build_root_flag_body() {
+ mkdir subdir
+ mkdir build
+ mkdir build/subdir
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+include("subdir/Kyuafile")
+atf_test_program{name="first"}
+EOF
+ echo 'invalid' >first
+ utils_cp_helper simple_all_pass build/first
+
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("in-subdir")
+atf_test_program{name="second"}
+EOF
+ echo 'invalid' >subdir/second
+ utils_cp_helper simple_some_fail build/subdir/second
+
+ cat >expout <<EOF
+subdir/second:fail
+subdir/second:pass
+first:pass
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list --build-root=build \
+ subdir first:pass
+}
+
+
+utils_test_case kyuafile_flag__no_args
+kyuafile_flag__no_args_body() {
+ cat >Kyuafile <<EOF
+This file is bogus but it is not loaded.
+EOF
+
+ cat >myfile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="sometest"}
+EOF
+ utils_cp_helper simple_all_pass sometest
+
+ cat >expout <<EOF
+sometest:pass
+sometest:skip
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list -k myfile
+ atf_check -s exit:0 -o file:expout -e empty kyua list --kyuafile=myfile
+}
+
+
+utils_test_case kyuafile_flag__some_args
+kyuafile_flag__some_args_body() {
+ cat >Kyuafile <<EOF
+This file is bogus but it is not loaded.
+EOF
+
+ cat >myfile <<EOF
+syntax(2)
+test_suite("hello-world")
+atf_test_program{name="sometest"}
+EOF
+ utils_cp_helper simple_all_pass sometest
+
+ cat >expout <<EOF
+sometest:pass (hello-world)
+sometest:skip (hello-world)
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list -v -k myfile sometest
+ atf_check -s exit:0 -o file:expout -e empty kyua list -v --kyuafile=myfile \
+ sometest
+}
+
+
+utils_test_case verbose_flag
+verbose_flag_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration-suite-1")
+atf_test_program{name="simple_all_pass"}
+plain_test_program{name="i_am_plain", timeout=654}
+include("subdir/Kyuafile")
+EOF
+ utils_cp_helper simple_all_pass .
+ touch i_am_plain
+
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration-suite-2")
+atf_test_program{name="metadata"}
+EOF
+ utils_cp_helper metadata subdir
+
+ cat >expout <<EOF
+simple_all_pass:pass (integration-suite-1)
+simple_all_pass:skip (integration-suite-1)
+i_am_plain:main (integration-suite-1)
+ timeout = 654
+subdir/metadata:many_properties (integration-suite-2)
+ allowed_architectures = some-architecture
+ allowed_platforms = some-platform
+ custom.no-meaning = I am a custom variable
+ description = A description with some padding
+ required_configs = var1 var2 var3
+ required_files = /my/file1 /some/other/file
+ required_programs = /nonexistent/bin3 bin1 bin2
+ required_user = root
+subdir/metadata:no_properties (integration-suite-2)
+subdir/metadata:one_property (integration-suite-2)
+ description = Does nothing but has one metadata property
+subdir/metadata:with_cleanup (integration-suite-2)
+ has_cleanup = true
+ timeout = 250
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list -v
+ atf_check -s exit:0 -o file:expout -e empty kyua list --verbose
+}
+
+
+utils_test_case no_test_program_match
+no_test_program_match_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_all_pass second
+
+ cat >experr <<EOF
+kyua: W: No test cases matched by the filter 'second'.
+EOF
+ atf_check -s exit:1 -o empty -e file:experr kyua list second
+}
+
+
+utils_test_case no_test_case_match
+no_test_case_match_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+EOF
+ utils_cp_helper simple_all_pass first
+
+ cat >experr <<EOF
+kyua: W: No test cases matched by the filter 'first:foobar'.
+EOF
+ atf_check -s exit:1 -o empty -e file:experr kyua list first:foobar
+}
+
+
+utils_test_case missing_kyuafile__no_args
+missing_kyuafile__no_args_body() {
+ cat >experr <<EOF
+kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua list
+}
+
+
+utils_test_case missing_kyuafile__test_program
+missing_kyuafile__test_program_body() {
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="unused"}
+EOF
+ utils_cp_helper simple_all_pass subdir/unused
+
+ cat >experr <<EOF
+kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua list subdir/unused
+}
+
+
+utils_test_case missing_kyuafile__subdir
+missing_kyuafile__subdir_body() {
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="unused"}
+EOF
+ utils_cp_helper simple_all_pass subdir/unused
+
+ cat >experr <<EOF
+kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua list subdir
+}
+
+
+utils_test_case bogus_kyuafile
+bogus_kyuafile_body() {
+ cat >Kyuafile <<EOF
+Hello, world.
+EOF
+ atf_check -s exit:2 -o empty \
+ -e match:"Load of 'Kyuafile' failed: .* Kyuafile:2:" kyua list
+}
+
+
+utils_test_case bogus_test_program
+bogus_test_program_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="crash_on_list"}
+atf_test_program{name="non_executable"}
+EOF
+ utils_cp_helper bad_test_program crash_on_list
+ echo 'I am not executable' >non_executable
+
+ cat >expout <<EOF
+crash_on_list:__test_cases_list__
+non_executable:__test_cases_list__
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua list
+}
+
+
+utils_test_case missing_test_program
+missing_test_program_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+include("subdir/Kyuafile")
+EOF
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="ok"}
+atf_test_program{name="i-am-missing"}
+EOF
+ echo 'I should not be touched because the Kyuafile is bogus' >subdir/ok
+
+# CHECK_STYLE_DISABLE
+ cat >experr <<EOF
+kyua: E: Load of 'Kyuafile' failed: .*Non-existent test program 'subdir/i-am-missing'.
+EOF
+# CHECK_STYLE_ENABLE
+ atf_check -s exit:2 -o empty -e "match:$(cat experr)" kyua list
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case no_args
+ atf_add_test_case one_arg__subdir
+ atf_add_test_case one_arg__test_case
+ atf_add_test_case one_arg__test_program
+ atf_add_test_case one_arg__invalid
+ atf_add_test_case many_args__ok
+ atf_add_test_case many_args__invalid
+ atf_add_test_case many_args__no_match__all
+ atf_add_test_case many_args__no_match__some
+
+ atf_add_test_case args_are_relative
+
+ atf_add_test_case only_load_used_test_programs
+
+ atf_add_test_case config_behavior
+
+ atf_add_test_case build_root_flag
+
+ atf_add_test_case kyuafile_flag__no_args
+ atf_add_test_case kyuafile_flag__some_args
+
+ atf_add_test_case verbose_flag
+
+ atf_add_test_case no_test_program_match
+ atf_add_test_case no_test_case_match
+
+ atf_add_test_case missing_kyuafile__no_args
+ atf_add_test_case missing_kyuafile__test_program
+ atf_add_test_case missing_kyuafile__subdir
+
+ atf_add_test_case bogus_kyuafile
+ atf_add_test_case bogus_test_program
+ atf_add_test_case missing_test_program
+}
diff --git a/integration/cmd_report_html_test.sh b/integration/cmd_report_html_test.sh
new file mode 100755
index 000000000000..9c9b4ba81c86
--- /dev/null
+++ b/integration/cmd_report_html_test.sh
@@ -0,0 +1,267 @@
+# Copyright 2012 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# Executes a mock test suite to generate data in the database.
+#
+# \param mock_env The value to store in a MOCK variable in the environment.
+# Use this to be able to differentiate executions by inspecting the
+# context of the output.
+# \param dbfile_name File to which to write the path to the generated database
+# file.
+run_tests() {
+ local mock_env="${1}"; shift
+ local dbfile_name="${1}"; shift
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="simple_all_pass"}
+atf_test_program{name="simple_some_fail"}
+atf_test_program{name="metadata"}
+EOF
+
+ utils_cp_helper simple_all_pass .
+ utils_cp_helper simple_some_fail .
+ utils_cp_helper metadata .
+ atf_check -s exit:1 -o save:stdout -e empty env MOCK="${mock_env}" kyua test
+ grep '^Results saved to ' stdout | cut -d ' ' -f 4 >"${dbfile_name}"
+ rm stdout
+
+ # Ensure the results of 'report-html' come from the database.
+ rm Kyuafile simple_all_pass simple_some_fail metadata
+}
+
+
+# Ensure a file has a set of strings.
+#
+# \param file The name of the file to check.
+# \param ... List of strings to check.
+check_in_file() {
+ local file="${1}"; shift
+
+ while [ ${#} -gt 0 ]; do
+ echo "Checking for presence of '${1}' in ${file}"
+ if grep "${1}" "${file}" >/dev/null; then
+ :
+ else
+ atf_fail "Test case output not found in HTML page ${file}"
+ fi
+ shift
+ done
+}
+
+
+# Ensure a file does not have a set of strings.
+#
+# \param file The name of the file to check.
+# \param ... List of strings to check.
+check_not_in_file() {
+ local file="${1}"; shift
+
+ while [ ${#} -gt 0 ]; do
+ echo "Checking for lack of '${1}' in ${file}"
+ if grep "${1}" "${file}" >/dev/null; then
+ atf_fail "Spurious test case output found in HTML page"
+ fi
+ shift
+ done
+}
+
+
+utils_test_case default_behavior__ok
+default_behavior__ok_body() {
+ run_tests "mock1" unused_dbfile_name
+
+ atf_check -s exit:0 -o ignore -e empty kyua report-html
+ for f in \
+ html/index.html \
+ html/context.html \
+ html/simple_all_pass_skip.html \
+ html/simple_some_fail_fail.html
+ do
+ test -f "${f}" || atf_fail "Missing ${f}"
+ done
+
+ atf_check -o match:"2 TESTS FAILING" cat html/index.html
+
+ check_in_file html/simple_all_pass_skip.html \
+ "This is the stdout of skip" "This is the stderr of skip"
+ check_not_in_file html/simple_all_pass_skip.html \
+ "This is the stdout of pass" "This is the stderr of pass" \
+ "This is the stdout of fail" "This is the stderr of fail" \
+ "Test case did not write anything to"
+
+ check_in_file html/simple_some_fail_fail.html \
+ "This is the stdout of fail" "This is the stderr of fail"
+ check_not_in_file html/simple_some_fail_fail.html \
+ "This is the stdout of pass" "This is the stderr of pass" \
+ "This is the stdout of skip" "This is the stderr of skip" \
+ "Test case did not write anything to"
+
+ check_in_file html/metadata_one_property.html \
+ "description = Does nothing but has one metadata property"
+ check_not_in_file html/metadata_one_property.html \
+ "allowed_architectures = some-architecture"
+
+ check_in_file html/metadata_many_properties.html \
+ "allowed_architectures = some-architecture"
+ check_not_in_file html/metadata_many_properties.html \
+ "description = Does nothing but has one metadata property"
+}
+
+
+utils_test_case default_behavior__no_store
+default_behavior__no_store_body() {
+ echo 'kyua: E: No previous results file found for test suite' \
+ "$(utils_test_suite_id)." >experr
+ atf_check -s exit:2 -o empty -e file:experr kyua report-html
+}
+
+
+utils_test_case results_file__explicit
+results_file__explicit_body() {
+ run_tests "mock1" dbfile_name1
+ run_tests "mock2" dbfile_name2
+
+ atf_check -s exit:0 -o ignore -e empty kyua report-html \
+ --results-file="$(cat dbfile_name1)"
+ grep "MOCK.*mock1" html/context.html || atf_fail "Invalid context in report"
+
+ rm -rf html
+ atf_check -s exit:0 -o ignore -e empty kyua report-html \
+ --results-file="$(cat dbfile_name2)"
+ grep "MOCK.*mock2" html/context.html || atf_fail "Invalid context in report"
+}
+
+
+utils_test_case results_file__not_found
+results_file__not_found_body() {
+ atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \
+ kyua report-html --results-file=foo
+}
+
+
+utils_test_case force__yes
+force__yes_body() {
+ run_tests "mock1" unused_dbfile_name
+
+ atf_check -s exit:0 -o ignore -e empty kyua report-html
+ test -f html/index.html || atf_fail "Expected file not created"
+ rm html/index.html
+ atf_check -s exit:0 -o ignore -e empty kyua report-html --force
+ test -f html/index.html || atf_fail "Expected file not created"
+}
+
+
+utils_test_case force__no
+force__no_body() {
+ run_tests "mock1" unused_dbfile_name
+
+ atf_check -s exit:0 -o ignore -e empty kyua report-html
+ test -f html/index.html || atf_fail "Expected file not created"
+ rm html/index.html
+
+cat >experr <<EOF
+kyua: E: Output directory 'html' already exists; maybe use --force?.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua report-html
+ test ! -f html/index.html || atf_fail "Not expected file created"
+}
+
+
+utils_test_case output__explicit
+output__explicit_body() {
+ run_tests "mock1" unused_dbfile_name
+
+ mkdir output
+ atf_check -s exit:0 -o ignore -e empty kyua report-html --output=output/foo
+ test ! -d html || atf_fail "Not expected directory created"
+ test -f output/foo/index.html || atf_fail "Expected file not created"
+}
+
+
+utils_test_case results_filter__ok
+results_filter__ok_body() {
+ run_tests "mock1" unused_dbfile_name
+
+ atf_check -s exit:0 -o ignore -e empty kyua report-html \
+ --results-filter=passed
+ for f in \
+ html/index.html \
+ html/context.html \
+ html/simple_all_pass_pass.html \
+ html/simple_some_fail_pass.html \
+ html/metadata_no_properties.html \
+ html/metadata_with_cleanup.html
+ do
+ test -f "${f}" || atf_fail "Missing ${f}"
+ done
+
+ atf_check -o match:"2 TESTS FAILING" cat html/index.html
+
+ check_in_file html/simple_all_pass_pass.html \
+ "This is the stdout of pass" "This is the stderr of pass"
+ check_not_in_file html/simple_all_pass_pass.html \
+ "This is the stdout of skip" "This is the stderr of skip" \
+ "This is the stdout of fail" "This is the stderr of fail" \
+ "Test case did not write anything to"
+
+ check_in_file html/simple_some_fail_pass.html \
+ "Test case did not write anything to stdout" \
+ "Test case did not write anything to stderr"
+ check_not_in_file html/simple_some_fail_pass.html \
+ "This is the stdout of pass" "This is the stderr of pass" \
+ "This is the stdout of skip" "This is the stderr of skip" \
+ "This is the stdout of fail" "This is the stderr of fail"
+}
+
+
+utils_test_case results_filter__invalid
+results_filter__invalid_body() {
+ echo "kyua: E: Unknown result type 'foo-bar'." >experr
+ atf_check -s exit:2 -o empty -e file:experr kyua report-html \
+ --results-filter=passed,foo-bar
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case default_behavior__ok
+ atf_add_test_case default_behavior__no_store
+
+ atf_add_test_case results_file__explicit
+ atf_add_test_case results_file__not_found
+
+ atf_add_test_case force__yes
+ atf_add_test_case force__no
+
+ atf_add_test_case output__explicit
+
+ atf_add_test_case results_filter__ok
+ atf_add_test_case results_filter__invalid
+}
diff --git a/integration/cmd_report_junit_test.sh b/integration/cmd_report_junit_test.sh
new file mode 100755
index 000000000000..af1a464f6004
--- /dev/null
+++ b/integration/cmd_report_junit_test.sh
@@ -0,0 +1,300 @@
+# Copyright 2014 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# Executes a mock test suite to generate data in the database.
+#
+# \param mock_env The value to store in a MOCK variable in the environment.
+# Use this to be able to differentiate executions by inspecting the
+# context of the output.
+# \param dbfile_name File to which to write the path to the generated database
+# file.
+run_tests() {
+ local mock_env="${1}"; shift
+ local dbfile_name="${1}"; shift
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="simple_all_pass"}
+EOF
+
+ utils_cp_helper simple_all_pass .
+ atf_check -s exit:0 -o save:stdout -e empty env MOCK="${mock_env}" kyua test
+ grep '^Results saved to ' stdout | cut -d ' ' -f 4 >"${dbfile_name}"
+ rm stdout
+
+ # Ensure the results of 'report-junit' come from the database.
+ rm Kyuafile simple_all_pass
+}
+
+
+# Removes the contents of a properties tag from stdout.
+strip_properties='awk "
+BEGIN { skip = 0; }
+
+/<\/properties>/ {
+ print \"</properties>\";
+ skip = 0;
+ next;
+}
+
+/<properties>/ {
+ print \"<properties>\";
+ print \"CONTENTS STRIPPED BY TEST\";
+ skip = 1;
+ next;
+}
+
+{ if (!skip) print; }"'
+
+
+utils_test_case default_behavior__ok
+default_behavior__ok_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1
+this should not be seen
+mock1 new line" unused_dbfile_name
+
+ cat >expout <<EOF
+<?xml version="1.0" encoding="iso-8859-1"?>
+<testsuite>
+<properties>
+CONTENTS STRIPPED BY TEST
+</properties>
+<testcase classname="simple_all_pass" name="pass" time="S.UUU">
+<system-out>This is the stdout of pass
+</system-out>
+<system-err>Test case metadata
+------------------
+
+allowed_architectures is empty
+allowed_platforms is empty
+description is empty
+has_cleanup = false
+is_exclusive = false
+required_configs is empty
+required_disk_space = 0
+required_files is empty
+required_memory = 0
+required_programs is empty
+required_user is empty
+timeout = 300
+
+Timing information
+------------------
+
+Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+End time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+Duration: S.UUUs
+
+Original stderr
+---------------
+
+This is the stderr of pass
+</system-err>
+</testcase>
+<testcase classname="simple_all_pass" name="skip" time="S.UUU">
+<skipped/>
+<system-out>This is the stdout of skip
+</system-out>
+<system-err>Skipped result details
+----------------------
+
+The reason for skipping is this
+
+Test case metadata
+------------------
+
+allowed_architectures is empty
+allowed_platforms is empty
+description is empty
+has_cleanup = false
+is_exclusive = false
+required_configs is empty
+required_disk_space = 0
+required_files is empty
+required_memory = 0
+required_programs is empty
+required_user is empty
+timeout = 300
+
+Timing information
+------------------
+
+Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+End time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+Duration: S.UUUs
+
+Original stderr
+---------------
+
+This is the stderr of skip
+</system-err>
+</testcase>
+</testsuite>
+EOF
+ atf_check -s exit:0 -o file:expout -e empty -x "kyua report-junit" \
+ "| ${strip_properties}"
+}
+
+
+utils_test_case default_behavior__no_store
+default_behavior__no_store_body() {
+ echo 'kyua: E: No previous results file found for test suite' \
+ "$(utils_test_suite_id)." >experr
+ atf_check -s exit:2 -o empty -e file:experr kyua report-junit
+}
+
+
+utils_test_case results_file__explicit
+results_file__explicit_body() {
+ run_tests "mock1" dbfile_name1
+ run_tests "mock2" dbfile_name2
+
+ atf_check -s exit:0 -o match:"MOCK.*mock1" -o not-match:"MOCK.*mock2" \
+ -e empty kyua report-junit --results-file="$(cat dbfile_name1)"
+ atf_check -s exit:0 -o not-match:"MOCK.*mock1" -o match:"MOCK.*mock2" \
+ -e empty kyua report-junit --results-file="$(cat dbfile_name2)"
+}
+
+
+utils_test_case results_file__not_found
+results_file__not_found_body() {
+ atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \
+ kyua report-junit --results-file=foo
+}
+
+
+utils_test_case output__explicit
+output__explicit_body() {
+ run_tests unused_mock unused_dbfile_name
+
+ cat >report <<EOF
+<?xml version="1.0" encoding="iso-8859-1"?>
+<testsuite>
+<properties>
+CONTENTS STRIPPED BY TEST
+</properties>
+<testcase classname="simple_all_pass" name="pass" time="S.UUU">
+<system-out>This is the stdout of pass
+</system-out>
+<system-err>Test case metadata
+------------------
+
+allowed_architectures is empty
+allowed_platforms is empty
+description is empty
+has_cleanup = false
+is_exclusive = false
+required_configs is empty
+required_disk_space = 0
+required_files is empty
+required_memory = 0
+required_programs is empty
+required_user is empty
+timeout = 300
+
+Timing information
+------------------
+
+Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+End time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+Duration: S.UUUs
+
+Original stderr
+---------------
+
+This is the stderr of pass
+</system-err>
+</testcase>
+<testcase classname="simple_all_pass" name="skip" time="S.UUU">
+<skipped/>
+<system-out>This is the stdout of skip
+</system-out>
+<system-err>Skipped result details
+----------------------
+
+The reason for skipping is this
+
+Test case metadata
+------------------
+
+allowed_architectures is empty
+allowed_platforms is empty
+description is empty
+has_cleanup = false
+is_exclusive = false
+required_configs is empty
+required_disk_space = 0
+required_files is empty
+required_memory = 0
+required_programs is empty
+required_user is empty
+timeout = 300
+
+Timing information
+------------------
+
+Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+End time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+Duration: S.UUUs
+
+Original stderr
+---------------
+
+This is the stderr of skip
+</system-err>
+</testcase>
+</testsuite>
+EOF
+
+ atf_check -s exit:0 -o file:report -e empty -x kyua report-junit \
+ --output=/dev/stdout "| ${strip_properties} | ${utils_strip_times}"
+ atf_check -s exit:0 -o empty -e save:stderr kyua report-junit \
+ --output=/dev/stderr
+ atf_check -s exit:0 -o file:report -x cat stderr \
+ "| ${strip_properties} | ${utils_strip_times}"
+
+ atf_check -s exit:0 -o empty -e empty kyua report-junit \
+ --output=my-file
+ atf_check -s exit:0 -o file:report -x cat my-file \
+ "| ${strip_properties} | ${utils_strip_times}"
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case default_behavior__ok
+ atf_add_test_case default_behavior__no_store
+
+ atf_add_test_case results_file__explicit
+ atf_add_test_case results_file__not_found
+
+ atf_add_test_case output__explicit
+}
diff --git a/integration/cmd_report_test.sh b/integration/cmd_report_test.sh
new file mode 100755
index 000000000000..18a5db386dfd
--- /dev/null
+++ b/integration/cmd_report_test.sh
@@ -0,0 +1,381 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# Executes a mock test suite to generate data in the database.
+#
+# \param mock_env The value to store in a MOCK variable in the environment.
+# Use this to be able to differentiate executions by inspecting the
+# context of the output.
+# \param dbfile_name File to which to write the path to the generated database
+# file.
+run_tests() {
+ local mock_env="${1}"; shift
+ local dbfile_name="${1}"; shift
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="simple_all_pass"}
+EOF
+
+ utils_cp_helper simple_all_pass .
+ atf_check -s exit:0 -o save:stdout -e empty env \
+ MOCK="${mock_env}" _='fake-value' kyua test
+ grep '^Results saved to ' stdout | cut -d ' ' -f 4 >"${dbfile_name}"
+ rm stdout
+
+ # Ensure the results of 'report' come from the database.
+ rm Kyuafile simple_all_pass
+}
+
+
+utils_test_case default_behavior__ok
+default_behavior__ok_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report
+
+ run_tests "mock2" dbfile_name2
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name2)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report
+}
+
+
+utils_test_case default_behavior__no_store
+default_behavior__no_store_body() {
+ echo 'kyua: E: No previous results file found for test suite' \
+ "$(utils_test_suite_id)." >experr
+ atf_check -s exit:2 -o empty -e file:experr kyua report
+}
+
+
+utils_test_case results_file__explicit
+results_file__explicit_body() {
+ run_tests "mock1" dbfile_name1
+ run_tests "mock2" dbfile_name2
+
+ atf_check -s exit:0 -o match:"MOCK=mock1" -o not-match:"MOCK=mock2" \
+ -e empty kyua report --results-file="$(cat dbfile_name1)" \
+ --verbose
+ atf_check -s exit:0 -o not-match:"MOCK=mock1" -o match:"MOCK=mock2" \
+ -e empty kyua report --results-file="$(cat dbfile_name2)" \
+ --verbose
+}
+
+
+utils_test_case results_file__not_found
+results_file__not_found_body() {
+ atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \
+ kyua report --results-file=foo
+}
+
+
+utils_test_case output__explicit
+output__explicit_body() {
+ run_tests unused_mock dbfile_name
+
+ cat >report <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+
+ atf_check -s exit:0 -o file:report -e empty -x kyua report \
+ --output=/dev/stdout "| ${utils_strip_times_but_not_ids}"
+ atf_check -s exit:0 -o empty -e save:stderr kyua report \
+ --output=/dev/stderr
+ atf_check -s exit:0 -o file:report -x cat stderr \
+ "| ${utils_strip_times_but_not_ids}"
+
+ atf_check -s exit:0 -o empty -e empty kyua report \
+ --output=my-file
+ atf_check -s exit:0 -o file:report -x cat my-file \
+ "| ${utils_strip_times_but_not_ids}"
+}
+
+
+utils_test_case filter__ok
+filter__ok_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 1 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ simple_all_pass:skip
+}
+
+
+utils_test_case filter__ok_passed_excluded_by_default
+filter__ok_passed_excluded_by_default_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ # Passed results are excluded by default so they are not displayed even if
+ # requested with a test case filter. This might be somewhat confusing...
+ cat >expout <<EOF
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 1 total, 0 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ simple_all_pass:pass
+ cat >expout <<EOF
+===> Passed tests
+simple_all_pass:pass -> passed [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 1 total, 0 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ --results-filter= simple_all_pass:pass
+}
+
+
+utils_test_case filter__no_match
+filter__no_match_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 1 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ cat >experr <<EOF
+kyua: W: No test cases matched by the filter 'first'.
+kyua: W: No test cases matched by the filter 'simple_all_pass:second'.
+EOF
+ atf_check -s exit:1 -o file:expout -e file:experr kyua report \
+ first simple_all_pass:skip simple_all_pass:second
+}
+
+
+utils_test_case verbose
+verbose_body() {
+ # Switch to the current directory using its physical location and update
+ # HOME accordingly. Otherwise, the test below where we compare the value
+ # of HOME in the output might fail if the path to HOME contains a symlink
+ # (as is the case in OS X when HOME points to the temporary directory.)
+ local real_cwd="$(pwd -P)"
+ cd "${real_cwd}"
+ HOME="${real_cwd}"
+
+ run_tests "mock1
+has multiple lines
+and terminates here" dbfile_name
+
+ cat >expout <<EOF
+===> Execution context
+Current directory: ${real_cwd}
+Environment variables:
+EOF
+ # $_ is a bash variable. To keep our tests stable, we override its value
+ # below to match the hardcoded value in run_tests.
+ env \
+ HOME="${real_cwd}" \
+ MOCK="mock1
+has multiple lines
+and terminates here" \
+ _='fake-value' \
+ "$(atf_get_srcdir)/helpers/dump_env" ' ' ' ' >>expout
+ cat >>expout <<EOF
+===> simple_all_pass:skip
+Result: skipped: The reason for skipping is this
+Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+End time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+Duration: S.UUUs
+
+Metadata:
+ allowed_architectures is empty
+ allowed_platforms is empty
+ description is empty
+ has_cleanup = false
+ is_exclusive = false
+ required_configs is empty
+ required_disk_space = 0
+ required_files is empty
+ required_memory = 0
+ required_programs is empty
+ required_user is empty
+ timeout = 300
+
+Standard output:
+This is the stdout of skip
+
+Standard error:
+This is the stderr of skip
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+End time: YYYY-MM-DDTHH:MM:SS.ssssssZ
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty -x kyua report --verbose \
+ "| ${utils_strip_times_but_not_ids}"
+}
+
+
+utils_test_case results_filter__empty
+results_filter__empty_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Passed tests
+simple_all_pass:pass -> passed [S.UUUs]
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report --results-filter=
+}
+
+
+utils_test_case results_filter__one
+results_filter__one_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Passed tests
+simple_all_pass:pass -> passed [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ --results-filter=passed
+}
+
+
+utils_test_case results_filter__multiple_all_match
+results_filter__multiple_all_match_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Passed tests
+simple_all_pass:pass -> passed [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ --results-filter=skipped,passed
+}
+
+
+utils_test_case results_filter__multiple_some_match
+results_filter__multiple_some_match_body() {
+ utils_install_times_wrapper
+
+ run_tests "mock1" dbfile_name1
+
+ cat >expout <<EOF
+===> Skipped tests
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+===> Summary
+Results read from $(cat dbfile_name1)
+Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed
+Total time: S.UUUs
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua report \
+ --results-filter=skipped,xfail,broken,failed
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case default_behavior__ok
+ atf_add_test_case default_behavior__no_store
+
+ atf_add_test_case results_file__explicit
+ atf_add_test_case results_file__not_found
+
+ atf_add_test_case filter__ok
+ atf_add_test_case filter__ok_passed_excluded_by_default
+ atf_add_test_case filter__no_match
+
+ atf_add_test_case verbose
+
+ atf_add_test_case output__explicit
+
+ atf_add_test_case results_filter__empty
+ atf_add_test_case results_filter__one
+ atf_add_test_case results_filter__multiple_all_match
+ atf_add_test_case results_filter__multiple_some_match
+}
diff --git a/integration/cmd_test_test.sh b/integration/cmd_test_test.sh
new file mode 100755
index 000000000000..bc8c62daf223
--- /dev/null
+++ b/integration/cmd_test_test.sh
@@ -0,0 +1,1071 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+utils_test_case one_test_program__all_pass
+one_test_program__all_pass_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="simple_all_pass"}
+EOF
+
+ cat >expout <<EOF
+simple_all_pass:pass -> passed [S.UUUs]
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+2/2 passed (0 failed)
+EOF
+
+ utils_cp_helper simple_all_pass .
+ atf_check -s exit:0 -o file:expout -e empty kyua test
+}
+
+
+utils_test_case one_test_program__some_fail
+one_test_program__some_fail_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="simple_some_fail"}
+EOF
+
+ cat >expout <<EOF
+simple_some_fail:fail -> failed: This fails on purpose [S.UUUs]
+simple_some_fail:pass -> passed [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+1/2 passed (1 failed)
+EOF
+
+ utils_cp_helper simple_some_fail .
+ atf_check -s exit:1 -o file:expout -e empty kyua test
+}
+
+
+utils_test_case many_test_programs__all_pass
+many_test_programs__all_pass_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+atf_test_program{name="third"}
+plain_test_program{name="fourth", required_files="/non-existent/foo"}
+EOF
+
+ cat >expout <<EOF
+first:pass -> passed [S.UUUs]
+first:skip -> skipped: The reason for skipping is this [S.UUUs]
+fourth:main -> skipped: Required file '/non-existent/foo' not found [S.UUUs]
+second:pass -> passed [S.UUUs]
+second:skip -> skipped: The reason for skipping is this [S.UUUs]
+third:pass -> passed [S.UUUs]
+third:skip -> skipped: The reason for skipping is this [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+7/7 passed (0 failed)
+EOF
+
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_all_pass second
+ utils_cp_helper simple_all_pass third
+ echo "not executed" >fourth; chmod +x fourth
+ atf_check -s exit:0 -o file:expout -e empty kyua test
+}
+
+
+utils_test_case many_test_programs__some_fail
+many_test_programs__some_fail_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+atf_test_program{name="third"}
+plain_test_program{name="fourth"}
+EOF
+
+ cat >expout <<EOF
+first:fail -> failed: This fails on purpose [S.UUUs]
+first:pass -> passed [S.UUUs]
+fourth:main -> failed: Returned non-success exit status 76 [S.UUUs]
+second:fail -> failed: This fails on purpose [S.UUUs]
+second:pass -> passed [S.UUUs]
+third:pass -> passed [S.UUUs]
+third:skip -> skipped: The reason for skipping is this [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+4/7 passed (3 failed)
+EOF
+
+ utils_cp_helper simple_some_fail first
+ utils_cp_helper simple_some_fail second
+ utils_cp_helper simple_all_pass third
+ echo '#! /bin/sh' >fourth
+ echo 'exit 76' >>fourth
+ chmod +x fourth
+ atf_check -s exit:1 -o file:expout -e empty kyua test
+}
+
+
+utils_test_case expect__all_pass
+expect__all_pass_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="expect_all_pass"}
+EOF
+
+# CHECK_STYLE_DISABLE
+ cat >expout <<EOF
+expect_all_pass:die -> expected_failure: This is the reason for death [S.UUUs]
+expect_all_pass:exit -> expected_failure: Exiting with correct code [S.UUUs]
+expect_all_pass:failure -> expected_failure: Oh no: Forced failure [S.UUUs]
+expect_all_pass:signal -> expected_failure: Exiting with correct signal [S.UUUs]
+expect_all_pass:timeout -> expected_failure: This times out [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+5/5 passed (0 failed)
+EOF
+# CHECK_STYLE_ENABLE
+
+ utils_cp_helper expect_all_pass .
+ atf_check -s exit:0 -o file:expout -e empty kyua test
+}
+
+
+utils_test_case expect__some_fail
+expect__some_fail_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="expect_some_fail"}
+EOF
+
+# CHECK_STYLE_DISABLE
+ cat >expout <<EOF
+expect_some_fail:die -> failed: Test case was expected to terminate abruptly but it continued execution [S.UUUs]
+expect_some_fail:exit -> failed: Test case expected to exit with code 12 but got code 34 [S.UUUs]
+expect_some_fail:failure -> failed: Test case was expecting a failure but none were raised [S.UUUs]
+expect_some_fail:pass -> passed [S.UUUs]
+expect_some_fail:signal -> failed: Test case expected to receive signal 15 but got 9 [S.UUUs]
+expect_some_fail:timeout -> failed: Test case was expected to hang but it continued execution [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+1/6 passed (5 failed)
+EOF
+# CHECK_STYLE_ENABLE
+
+ utils_cp_helper expect_some_fail .
+ atf_check -s exit:1 -o file:expout -e empty kyua test
+}
+
+
+utils_test_case premature_exit
+premature_exit_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="bogus_test_cases"}
+EOF
+
+# CHECK_STYLE_DISABLE
+ cat >expout <<EOF
+bogus_test_cases:die -> broken: Premature exit; test case received signal 9 [S.UUUs]
+bogus_test_cases:exit -> broken: Premature exit; test case exited with code 0 [S.UUUs]
+bogus_test_cases:pass -> passed [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+1/3 passed (2 failed)
+EOF
+# CHECK_STYLE_ENABLE
+
+ utils_cp_helper bogus_test_cases .
+ atf_check -s exit:1 -o file:expout -e empty kyua test
+}
+
+
+utils_test_case no_args
+no_args_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="simple_all_pass"}
+include("subdir/Kyuafile")
+EOF
+ utils_cp_helper metadata .
+ utils_cp_helper simple_all_pass .
+
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration2")
+atf_test_program{name="simple_some_fail"}
+EOF
+ utils_cp_helper simple_some_fail subdir
+
+ cat >expout <<EOF
+simple_all_pass:pass -> passed [S.UUUs]
+simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+subdir/simple_some_fail:fail -> failed: This fails on purpose [S.UUUs]
+subdir/simple_some_fail:pass -> passed [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+3/4 passed (1 failed)
+EOF
+ atf_check -s exit:1 -o file:expout -e empty kyua test
+}
+
+
+utils_test_case one_arg__subdir
+one_arg__subdir_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+include("subdir/Kyuafile")
+EOF
+
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("in-subdir")
+atf_test_program{name="simple_all_pass"}
+EOF
+ utils_cp_helper simple_all_pass subdir
+
+# CHECK_STYLE_DISABLE
+ cat >expout <<EOF
+subdir/simple_all_pass:pass -> passed [S.UUUs]
+subdir/simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+2/2 passed (0 failed)
+EOF
+# CHECK_STYLE_ENABLE
+ atf_check -s exit:0 -o file:expout -e empty kyua test subdir
+}
+
+
+utils_test_case one_arg__test_case
+one_arg__test_case_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_all_pass second
+
+ cat >expout <<EOF
+first:skip -> skipped: The reason for skipping is this [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+1/1 passed (0 failed)
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua test first:skip
+}
+
+
+utils_test_case one_arg__test_program
+one_arg__test_program_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_some_fail second
+
+ cat >expout <<EOF
+second:fail -> failed: This fails on purpose [S.UUUs]
+second:pass -> passed [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+1/2 passed (1 failed)
+EOF
+ atf_check -s exit:1 -o file:expout -e empty kyua test second
+}
+
+
+utils_test_case one_arg__invalid
+one_arg__invalid_body() {
+cat >experr <<EOF
+kyua: E: Test case component in 'foo:' is empty.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua test foo:
+
+cat >experr <<EOF
+kyua: E: Program name '/a/b' must be relative to the test suite, not absolute.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua test /a/b
+}
+
+
+utils_test_case many_args__ok
+many_args__ok_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+include("subdir/Kyuafile")
+atf_test_program{name="first"}
+EOF
+ utils_cp_helper simple_all_pass first
+
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("in-subdir")
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_some_fail subdir/second
+
+ cat >expout <<EOF
+first:pass -> passed [S.UUUs]
+subdir/second:fail -> failed: This fails on purpose [S.UUUs]
+subdir/second:pass -> passed [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+2/3 passed (1 failed)
+EOF
+ atf_check -s exit:1 -o file:expout -e empty kyua test subdir first:pass
+}
+
+
+utils_test_case many_args__invalid
+many_args__invalid_body() {
+cat >experr <<EOF
+kyua: E: Program name component in ':badbad' is empty.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua test this-is-ok :badbad
+
+cat >experr <<EOF
+kyua: E: Program name '/foo' must be relative to the test suite, not absolute.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua test this-is-ok /foo
+}
+
+
+utils_test_case many_args__no_match__all
+many_args__no_match__all_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_all_pass second
+
+ cat >expout <<EOF
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+EOF
+ cat >experr <<EOF
+kyua: W: No test cases matched by the filter 'first1'.
+EOF
+ atf_check -s exit:1 -o file:expout -e file:experr kyua test first1
+}
+
+
+utils_test_case many_args__no_match__some
+many_args__no_match__some_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("top-level")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+atf_test_program{name="third"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_all_pass second
+ utils_cp_helper simple_some_fail third
+
+ cat >expout <<EOF
+first:pass -> passed [S.UUUs]
+first:skip -> skipped: The reason for skipping is this [S.UUUs]
+third:fail -> failed: This fails on purpose [S.UUUs]
+third:pass -> passed [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+3/4 passed (1 failed)
+EOF
+
+ cat >experr <<EOF
+kyua: W: No test cases matched by the filter 'fifth'.
+kyua: W: No test cases matched by the filter 'fourth'.
+EOF
+ atf_check -s exit:1 -o file:expout -e file:experr kyua test first fourth \
+ third fifth
+}
+
+
+utils_test_case args_are_relative
+args_are_relative_body() {
+ utils_install_stable_test_wrapper
+
+ mkdir root
+ cat >root/Kyuafile <<EOF
+syntax(2)
+test_suite("integration-1")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+include("subdir/Kyuafile")
+EOF
+ utils_cp_helper simple_all_pass root/first
+ utils_cp_helper simple_some_fail root/second
+
+ mkdir root/subdir
+ cat >root/subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration-2")
+atf_test_program{name="third"}
+atf_test_program{name="fourth"}
+EOF
+ utils_cp_helper simple_all_pass root/subdir/third
+ utils_cp_helper simple_some_fail root/subdir/fourth
+
+ cat >expout <<EOF
+first:pass -> passed [S.UUUs]
+first:skip -> skipped: The reason for skipping is this [S.UUUs]
+subdir/fourth:fail -> failed: This fails on purpose [S.UUUs]
+
+Results file id is $(utils_results_id root)
+Results saved to $(utils_results_file root)
+
+2/3 passed (1 failed)
+EOF
+ atf_check -s exit:1 -o file:expout -e empty kyua test \
+ -k "$(pwd)/root/Kyuafile" first subdir/fourth:fail
+}
+
+
+utils_test_case only_load_used_test_programs
+only_load_used_test_programs_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+atf_test_program{name="second"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper bad_test_program second
+
+ cat >expout <<EOF
+first:pass -> passed [S.UUUs]
+first:skip -> skipped: The reason for skipping is this [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+2/2 passed (0 failed)
+EOF
+ CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE
+ atf_check -s exit:0 -o file:expout -e empty kyua test first
+ if [ -f "${CREATE_COOKIE}" ]; then
+ atf_fail "An unmatched test case has been executed, which harms" \
+ "performance"
+ fi
+}
+
+
+utils_test_case config_behavior
+config_behavior_body() {
+ cat >"my-config" <<EOF
+syntax(2)
+test_suites.suite1["the-variable"] = "value1"
+test_suites.suite2["the-variable"] = "override me"
+EOF
+
+ cat >Kyuafile <<EOF
+syntax(2)
+atf_test_program{name="config1", test_suite="suite1"}
+atf_test_program{name="config2", test_suite="suite2"}
+atf_test_program{name="config3", test_suite="suite3"}
+EOF
+ utils_cp_helper config config1
+ utils_cp_helper config config2
+ utils_cp_helper config config3
+
+ atf_check -s exit:1 -o save:stdout -e empty \
+ kyua -c my-config -v test_suites.suite2.the-variable=value2 test
+ atf_check -s exit:0 -o ignore -e empty \
+ grep 'config1:get_variable.*failed' stdout
+ atf_check -s exit:0 -o ignore -e empty \
+ grep 'config2:get_variable.*passed' stdout
+ atf_check -s exit:0 -o ignore -e empty \
+ grep 'config3:get_variable.*skipped' stdout
+
+ CONFIG_VAR_FILE="$(pwd)/cookie"; export CONFIG_VAR_FILE
+ if [ -f "${CONFIG_VAR_FILE}" ]; then
+ atf_fail "Cookie file already created; test case list may have gotten" \
+ "a bad configuration"
+ fi
+ atf_check -s exit:1 -o ignore -e empty kyua -c my-config test config1
+ [ -f "${CONFIG_VAR_FILE}" ] || \
+ atf_fail "Cookie file not created; test case list did not get" \
+ "configuration variables"
+ value="$(cat "${CONFIG_VAR_FILE}")"
+ [ "${value}" = "value1" ] || \
+ atf_fail "Invalid value (${value}) in cookie file; test case list did" \
+ "not get the correct configuration variables"
+}
+
+
+utils_test_case store_contents
+store_contents_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+atf_test_program{name="some-program", test_suite="suite1"}
+EOF
+ utils_cp_helper simple_some_fail some-program
+ cat >expout <<EOF
+some-program:fail -> failed: This fails on purpose [S.UUUs]
+some-program:pass -> passed [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+1/2 passed (1 failed)
+EOF
+
+ atf_check -s exit:1 -o file:expout -e empty kyua test
+
+cat >expout <<EOF
+some-program,fail,failed,This fails on purpose
+some-program,pass,passed,NULL
+EOF
+ atf_check -s exit:0 -o file:expout -e empty \
+ kyua db-exec --no-headers \
+ "SELECT " \
+ " test_programs.relative_path, test_cases.name, " \
+ " test_results.result_type, test_results.result_reason " \
+ "FROM test_programs " \
+ " JOIN test_cases " \
+ " ON test_programs.test_program_id = test_cases.test_program_id " \
+ " JOIN test_results " \
+ " ON test_cases.test_case_id = test_results.test_case_id " \
+ "ORDER BY test_programs.relative_path, test_cases.name"
+}
+
+
+utils_test_case results_file__ok
+results_file__ok_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+atf_test_program{name="config1", test_suite="suite1"}
+EOF
+ utils_cp_helper config config1
+
+ atf_check -s exit:0 -o ignore -e empty kyua test -r foo1.db
+ test -f foo1.db || atf_fail "-s did not work"
+ atf_check -s exit:0 -o ignore -e empty kyua test --results-file=foo2.db
+ test -f foo2.db || atf_fail "--results-file did not work"
+ test ! -f .kyua/store.db || atf_fail "Default database created"
+}
+
+
+utils_test_case results_file__fail
+results_file__fail_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+atf_test_program{name="config1", test_suite="suite1"}
+EOF
+ utils_cp_helper config config1
+
+ atf_check -s exit:3 -o empty -e match:"Invalid.*--results-file" \
+ kyua test --results-file=
+}
+
+
+utils_test_case results_file__reuse
+results_file__reuse_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+atf_test_program{name="simple_all_pass", test_suite="integration"}
+EOF
+ utils_cp_helper simple_all_pass .
+ atf_check -s exit:0 -o ignore -e empty kyua test -r results.db
+
+ atf_check -s exit:2 -o empty -e match:"results.db already exists" \
+ kyua test --results-file="results.db"
+}
+
+
+utils_test_case build_root_flag
+build_root_flag_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+include("subdir/Kyuafile")
+EOF
+
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="second"}
+atf_test_program{name="third"}
+EOF
+
+ cat >expout <<EOF
+first:pass -> passed [S.UUUs]
+first:skip -> skipped: The reason for skipping is this [S.UUUs]
+subdir/second:pass -> passed [S.UUUs]
+subdir/second:skip -> skipped: The reason for skipping is this [S.UUUs]
+subdir/third:pass -> passed [S.UUUs]
+subdir/third:skip -> skipped: The reason for skipping is this [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+6/6 passed (0 failed)
+EOF
+
+ mkdir build
+ mkdir build/subdir
+ utils_cp_helper simple_all_pass build/first
+ utils_cp_helper simple_all_pass build/subdir/second
+ utils_cp_helper simple_all_pass build/subdir/third
+
+ atf_check -s exit:0 -o file:expout -e empty kyua test --build-root=build
+}
+
+
+utils_test_case kyuafile_flag__no_args
+kyuafile_flag__no_args_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+This file is bogus but it is not loaded.
+EOF
+
+ cat >myfile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="sometest"}
+EOF
+ utils_cp_helper simple_all_pass sometest
+
+ cat >expout <<EOF
+sometest:pass -> passed [S.UUUs]
+sometest:skip -> skipped: The reason for skipping is this [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+2/2 passed (0 failed)
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua test -k myfile
+ atf_check -s exit:0 -o file:expout -e empty kyua test --kyuafile=myfile
+}
+
+
+utils_test_case kyuafile_flag__some_args
+kyuafile_flag__some_args_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+This file is bogus but it is not loaded.
+EOF
+
+ cat >myfile <<EOF
+syntax(2)
+test_suite("hello-world")
+atf_test_program{name="sometest"}
+EOF
+ utils_cp_helper simple_all_pass sometest
+
+ cat >expout <<EOF
+sometest:pass -> passed [S.UUUs]
+sometest:skip -> skipped: The reason for skipping is this [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+2/2 passed (0 failed)
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua test -k myfile sometest
+ cat >expout <<EOF
+sometest:pass -> passed [S.UUUs]
+sometest:skip -> skipped: The reason for skipping is this [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+2/2 passed (0 failed)
+EOF
+ atf_check -s exit:0 -o file:expout -e empty kyua test --kyuafile=myfile \
+ sometest
+}
+
+
+utils_test_case interrupt
+interrupt_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="interrupts"}
+EOF
+ utils_cp_helper interrupts .
+
+ kyua \
+ -v test_suites.integration.body-cookie="$(pwd)/body" \
+ -v test_suites.integration.cleanup-cookie="$(pwd)/cleanup" \
+ test >stdout 2>stderr &
+ pid=${!}
+ echo "Kyua subprocess is PID ${pid}"
+
+ while [ ! -f body ]; do
+ echo "Waiting for body to start"
+ sleep 1
+ done
+ echo "Body started"
+ sleep 1
+
+ echo "Sending INT signal to ${pid}"
+ kill -INT ${pid}
+ echo "Waiting for process ${pid} to exit"
+ wait ${pid}
+ ret=${?}
+ sed -e 's,^,kyua stdout:,' stdout
+ sed -e 's,^,kyua stderr:,' stderr
+ echo "Process ${pid} exited"
+ [ ${ret} -ne 0 ] || atf_fail 'No error code reported'
+
+ [ -f cleanup ] || atf_fail 'Cleanup part not executed after signal'
+ atf_expect_pass
+
+ atf_check -s exit:0 -o ignore -e empty grep 'Signal caught' stderr
+ atf_check -s exit:0 -o ignore -e empty \
+ grep 'kyua: E: Interrupted by signal' stderr
+}
+
+
+utils_test_case exclusive_tests
+exclusive_tests_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+EOF
+ for i in $(seq 100); do
+ echo 'plain_test_program{name="race", is_exclusive=true}' >>Kyuafile
+ done
+ utils_cp_helper race .
+
+ atf_check \
+ -s exit:0 \
+ -o match:"100/100 passed" \
+ kyua \
+ -v parallelism=20 \
+ -v test_suites.integration.shared_file="$(pwd)/shared_file" \
+ test
+}
+
+
+utils_test_case no_test_program_match
+no_test_program_match_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+EOF
+ utils_cp_helper simple_all_pass first
+ utils_cp_helper simple_all_pass second
+
+ cat >expout <<EOF
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+EOF
+ cat >experr <<EOF
+kyua: W: No test cases matched by the filter 'second'.
+EOF
+ atf_check -s exit:1 -o file:expout -e file:experr kyua test second
+}
+
+
+utils_test_case no_test_case_match
+no_test_case_match_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="first"}
+EOF
+ utils_cp_helper simple_all_pass first
+
+ cat >expout <<EOF
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+EOF
+ cat >experr <<EOF
+kyua: W: No test cases matched by the filter 'first:foobar'.
+EOF
+ atf_check -s exit:1 -o file:expout -e file:experr kyua test first:foobar
+}
+
+
+utils_test_case missing_kyuafile__no_args
+missing_kyuafile__no_args_body() {
+ cat >experr <<EOF
+kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua test
+}
+
+
+utils_test_case missing_kyuafile__test_program
+missing_kyuafile__test_program_body() {
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="unused"}
+EOF
+ utils_cp_helper simple_all_pass subdir/unused
+
+ cat >experr <<EOF
+kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua test subdir/unused
+}
+
+
+utils_test_case missing_kyuafile__subdir
+missing_kyuafile__subdir_body() {
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="unused"}
+EOF
+ utils_cp_helper simple_all_pass subdir/unused
+
+ cat >experr <<EOF
+kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found.
+EOF
+ atf_check -s exit:2 -o empty -e file:experr kyua test subdir
+}
+
+
+utils_test_case bogus_config
+bogus_config_body() {
+ mkdir .kyua
+ cat >"${HOME}/.kyua/kyua.conf" <<EOF
+Hello, world.
+EOF
+
+ file_re='.*\.kyua/kyua.conf'
+ atf_check -s exit:2 -o empty \
+ -e match:"^kyua: E: Load of '${file_re}' failed: Failed to load Lua" \
+ kyua test
+}
+
+
+utils_test_case bogus_kyuafile
+bogus_kyuafile_body() {
+ cat >Kyuafile <<EOF
+Hello, world.
+EOF
+ atf_check -s exit:2 -o empty \
+ -e match:"Load of 'Kyuafile' failed: .* Kyuafile:2:" kyua list
+}
+
+
+utils_test_case bogus_test_program
+bogus_test_program_body() {
+ utils_install_stable_test_wrapper
+
+ cat >Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="crash_on_list"}
+atf_test_program{name="non_executable"}
+EOF
+ utils_cp_helper bad_test_program crash_on_list
+ echo 'I am not executable' >non_executable
+
+# CHECK_STYLE_DISABLE
+ cat >expout <<EOF
+crash_on_list:__test_cases_list__ -> broken: Invalid header for test case list; expecting Content-Type for application/X-atf-tp version 1, got '' [S.UUUs]
+non_executable:__test_cases_list__ -> broken: Permission denied to run test program [S.UUUs]
+
+Results file id is $(utils_results_id)
+Results saved to $(utils_results_file)
+
+0/2 passed (2 failed)
+EOF
+# CHECK_STYLE_ENABLE
+ atf_check -s exit:1 -o file:expout -e empty kyua test
+}
+
+
+utils_test_case missing_test_program
+missing_test_program_body() {
+ cat >Kyuafile <<EOF
+syntax(2)
+include("subdir/Kyuafile")
+EOF
+ mkdir subdir
+ cat >subdir/Kyuafile <<EOF
+syntax(2)
+test_suite("integration")
+atf_test_program{name="ok"}
+atf_test_program{name="i-am-missing"}
+EOF
+ echo 'I should not be touched because the Kyuafile is bogus' >subdir/ok
+
+# CHECK_STYLE_DISABLE
+ cat >experr <<EOF
+kyua: E: Load of 'Kyuafile' failed: .*Non-existent test program 'subdir/i-am-missing'.
+EOF
+# CHECK_STYLE_ENABLE
+ atf_check -s exit:2 -o empty -e "match:$(cat experr)" kyua list
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case one_test_program__all_pass
+ atf_add_test_case one_test_program__some_fail
+ atf_add_test_case many_test_programs__all_pass
+ atf_add_test_case many_test_programs__some_fail
+ atf_add_test_case expect__all_pass
+ atf_add_test_case expect__some_fail
+ atf_add_test_case premature_exit
+
+ atf_add_test_case no_args
+ atf_add_test_case one_arg__subdir
+ atf_add_test_case one_arg__test_case
+ atf_add_test_case one_arg__test_program
+ atf_add_test_case one_arg__invalid
+ atf_add_test_case many_args__ok
+ atf_add_test_case many_args__invalid
+ atf_add_test_case many_args__no_match__all
+ atf_add_test_case many_args__no_match__some
+
+ atf_add_test_case args_are_relative
+
+ atf_add_test_case only_load_used_test_programs
+
+ atf_add_test_case config_behavior
+
+ atf_add_test_case store_contents
+ atf_add_test_case results_file__ok
+ atf_add_test_case results_file__fail
+ atf_add_test_case results_file__reuse
+
+ atf_add_test_case build_root_flag
+
+ atf_add_test_case kyuafile_flag__no_args
+ atf_add_test_case kyuafile_flag__some_args
+
+ atf_add_test_case interrupt
+
+ atf_add_test_case exclusive_tests
+
+ atf_add_test_case no_test_program_match
+ atf_add_test_case no_test_case_match
+
+ atf_add_test_case missing_kyuafile__no_args
+ atf_add_test_case missing_kyuafile__test_program
+ atf_add_test_case missing_kyuafile__subdir
+
+ atf_add_test_case bogus_config
+ atf_add_test_case bogus_kyuafile
+ atf_add_test_case bogus_test_program
+ atf_add_test_case missing_test_program
+}
diff --git a/integration/global_test.sh b/integration/global_test.sh
new file mode 100755
index 000000000000..9cc8be2d1dec
--- /dev/null
+++ b/integration/global_test.sh
@@ -0,0 +1,146 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+utils_test_case no_args
+no_args_body() {
+ cat >experr <<EOF
+Usage error: No command provided.
+Type 'kyua help' for usage information.
+EOF
+
+ atf_check -s exit:3 -o empty -e file:experr kyua
+}
+
+
+utils_test_case unknown_option
+unknown_option_body() {
+ cat >experr <<EOF
+Usage error: Unknown option --this_is_unknown.
+Type 'kyua help' for usage information.
+EOF
+
+ atf_check -s exit:3 -o empty -e file:experr kyua --this_is_unknown
+}
+
+
+utils_test_case unknown_command
+unknown_command_body() {
+ cat >experr <<EOF
+Usage error: Unknown command 'i_am_not_known'.
+Type 'kyua help' for usage information.
+EOF
+
+ atf_check -s exit:3 -o empty -e file:experr kyua i_am_not_known
+}
+
+
+utils_test_case logfile__default
+logfile__default_body() {
+ atf_check -s exit:0 test ! -d .kyua/logs/
+ atf_check -s exit:3 -o empty -e ignore kyua
+ atf_check -s exit:0 test -d .kyua/logs/
+}
+
+
+utils_test_case logfile__override
+logfile__override_body() {
+ atf_check -s exit:0 test ! -f test.log
+ atf_check -s exit:3 -o empty -e ignore kyua --logfile=test.log
+
+ atf_check -s exit:0 test ! -d .kyua/logs/
+ atf_check -s exit:0 test -f test.log
+
+ grep ' E .* No command provided' test.log || atf_fail "Log file does" \
+ "contain required message"
+}
+
+
+utils_test_case loglevel__default
+loglevel__default_body() {
+ atf_check -s exit:0 test ! -f test.log
+ atf_check -s exit:3 -o empty -e ignore kyua --logfile=test.log
+
+ atf_check -s exit:0 test ! -d .kyua/logs/
+ atf_check -s exit:0 test -f test.log
+
+ grep ' E .* No command provided' test.log || atf_fail "Log file does" \
+ "contain required message"
+ if grep ' D ' test.log; then
+ atf_fail "Log file contains debug messages but it should not"
+ fi
+}
+
+
+utils_test_case loglevel__lower
+loglevel__lower_body() {
+ atf_check -s exit:0 test ! -f test.log
+ atf_check -s exit:3 -o empty -e ignore kyua --logfile=test.log \
+ --loglevel=warning
+
+ atf_check -s exit:0 test ! -d .kyua/logs/
+ atf_check -s exit:0 test -f test.log
+
+ grep ' E .* No command provided' test.log || atf_fail "Log file does" \
+ "contain required message"
+ if grep ' I ' test.log; then
+ atf_fail "Log file contains info messages but it should not"
+ fi
+}
+
+
+utils_test_case loglevel__higher
+loglevel__higher_body() {
+ atf_check -s exit:0 test ! -f test.log
+ atf_check -s exit:3 -o empty -e ignore kyua --logfile=test.log \
+ --loglevel=debug
+
+ atf_check -s exit:0 test ! -d .kyua/logs/
+ atf_check -s exit:0 test -f test.log
+
+ grep ' E .* No command provided' test.log || atf_fail "Log file does" \
+ "contain required message"
+ grep ' D ' test.log || atf_fail "Log file does not contain debug messages"
+}
+
+
+atf_init_test_cases() {
+ atf_add_test_case no_args
+ atf_add_test_case unknown_option
+ atf_add_test_case unknown_command
+
+ atf_add_test_case logfile__default
+ atf_add_test_case logfile__override
+
+ atf_add_test_case loglevel__default
+ atf_add_test_case loglevel__lower
+ atf_add_test_case loglevel__higher
+
+ # Tests for the global configuration-related flags are found in the
+ # cmd_config_test test program.
+}
diff --git a/integration/helpers/.gitignore b/integration/helpers/.gitignore
new file mode 100644
index 000000000000..11634bcfeb85
--- /dev/null
+++ b/integration/helpers/.gitignore
@@ -0,0 +1,11 @@
+bad_test_program
+bogus_test_cases
+config
+dump_env
+expect_all_pass
+expect_some_fail
+interrupts
+metadata
+race
+simple_all_pass
+simple_some_fail
diff --git a/integration/helpers/Makefile.am.inc b/integration/helpers/Makefile.am.inc
new file mode 100644
index 000000000000..d835aba51c67
--- /dev/null
+++ b/integration/helpers/Makefile.am.inc
@@ -0,0 +1,90 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+if WITH_ATF
+tests_integration_helpersdir = $(pkgtestsdir)/integration/helpers
+
+tests_integration_helpers_PROGRAMS = integration/helpers/bad_test_program
+integration_helpers_bad_test_program_SOURCES = \
+ integration/helpers/bad_test_program.cpp
+
+tests_integration_helpers_PROGRAMS += integration/helpers/bogus_test_cases
+integration_helpers_bogus_test_cases_SOURCES = \
+ integration/helpers/bogus_test_cases.cpp
+integration_helpers_bogus_test_cases_CXXFLAGS = $(ATF_CXX_CFLAGS)
+integration_helpers_bogus_test_cases_LDADD = $(ATF_CXX_LIBS)
+
+tests_integration_helpers_PROGRAMS += integration/helpers/config
+integration_helpers_config_SOURCES = integration/helpers/config.cpp
+integration_helpers_config_CXXFLAGS = $(ATF_CXX_CFLAGS)
+integration_helpers_config_LDADD = $(ATF_CXX_LIBS)
+
+tests_integration_helpers_PROGRAMS += integration/helpers/dump_env
+integration_helpers_dump_env_SOURCES = integration/helpers/dump_env.cpp
+integration_helpers_dump_env_CXXFLAGS = $(UTILS_CFLAGS)
+integration_helpers_dump_env_LDADD = $(UTILS_LIBS)
+
+tests_integration_helpers_PROGRAMS += integration/helpers/expect_all_pass
+integration_helpers_expect_all_pass_SOURCES = \
+ integration/helpers/expect_all_pass.cpp
+integration_helpers_expect_all_pass_CXXFLAGS = $(ATF_CXX_CFLAGS)
+integration_helpers_expect_all_pass_LDADD = $(ATF_CXX_LIBS)
+
+tests_integration_helpers_PROGRAMS += integration/helpers/expect_some_fail
+integration_helpers_expect_some_fail_SOURCES = \
+ integration/helpers/expect_some_fail.cpp
+integration_helpers_expect_some_fail_CXXFLAGS = $(ATF_CXX_CFLAGS)
+integration_helpers_expect_some_fail_LDADD = $(ATF_CXX_LIBS)
+
+tests_integration_helpers_PROGRAMS += integration/helpers/interrupts
+integration_helpers_interrupts_SOURCES = integration/helpers/interrupts.cpp
+integration_helpers_interrupts_CXXFLAGS = $(ATF_CXX_CFLAGS)
+integration_helpers_interrupts_LDADD = $(ATF_CXX_LIBS)
+
+tests_integration_helpers_PROGRAMS += integration/helpers/metadata
+integration_helpers_metadata_SOURCES = integration/helpers/metadata.cpp
+integration_helpers_metadata_CXXFLAGS = $(ATF_CXX_CFLAGS)
+integration_helpers_metadata_LDADD = $(ATF_CXX_LIBS)
+
+tests_integration_helpers_PROGRAMS += integration/helpers/race
+integration_helpers_race_SOURCES = integration/helpers/race.cpp
+integration_helpers_race_CXXFLAGS = $(UTILS_CFLAGS)
+integration_helpers_race_LDADD = $(UTILS_LIBS)
+
+tests_integration_helpers_PROGRAMS += integration/helpers/simple_all_pass
+integration_helpers_simple_all_pass_SOURCES = \
+ integration/helpers/simple_all_pass.cpp
+integration_helpers_simple_all_pass_CXXFLAGS = $(ATF_CXX_CFLAGS)
+integration_helpers_simple_all_pass_LDADD = $(ATF_CXX_LIBS)
+
+tests_integration_helpers_PROGRAMS += integration/helpers/simple_some_fail
+integration_helpers_simple_some_fail_SOURCES = \
+ integration/helpers/simple_some_fail.cpp
+integration_helpers_simple_some_fail_CXXFLAGS = $(ATF_CXX_CFLAGS)
+integration_helpers_simple_some_fail_LDADD = $(ATF_CXX_LIBS)
+endif
diff --git a/integration/helpers/bad_test_program.cpp b/integration/helpers/bad_test_program.cpp
new file mode 100644
index 000000000000..210709ee8976
--- /dev/null
+++ b/integration/helpers/bad_test_program.cpp
@@ -0,0 +1,50 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+
+
+int
+main(void)
+{
+ std::cerr << "This is not a valid test program!\n";
+
+ const char* cookie = std::getenv("CREATE_COOKIE");
+ if (cookie != NULL && std::strlen(cookie) > 0) {
+ std::ofstream file(cookie);
+ if (!file)
+ std::abort();
+ file << "Cookie file\n";
+ file.close();
+ }
+
+ return EXIT_SUCCESS;
+}
diff --git a/integration/helpers/bogus_test_cases.cpp b/integration/helpers/bogus_test_cases.cpp
new file mode 100644
index 000000000000..1a7c27031e1b
--- /dev/null
+++ b/integration/helpers/bogus_test_cases.cpp
@@ -0,0 +1,64 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+extern "C" {
+#include <signal.h>
+#include <unistd.h>
+}
+
+#include <cstdlib>
+
+#include <atf-c++.hpp>
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(die);
+ATF_TEST_CASE_BODY(die)
+{
+ ::kill(::getpid(), SIGKILL);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(exit);
+ATF_TEST_CASE_BODY(exit)
+{
+ std::exit(EXIT_SUCCESS);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(pass);
+ATF_TEST_CASE_BODY(pass)
+{
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, die);
+ ATF_ADD_TEST_CASE(tcs, exit);
+ ATF_ADD_TEST_CASE(tcs, pass);
+}
diff --git a/integration/helpers/config.cpp b/integration/helpers/config.cpp
new file mode 100644
index 000000000000..aa4fef291725
--- /dev/null
+++ b/integration/helpers/config.cpp
@@ -0,0 +1,58 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <cstdlib>
+
+#include <atf-c++.hpp>
+
+
+ATF_TEST_CASE(get_variable);
+ATF_TEST_CASE_HEAD(get_variable)
+{
+ const char* output = ::getenv("CONFIG_VAR_FILE");
+ if (output == NULL) {
+ set_md_var("require.config", "the-variable");
+ } else {
+ if (has_config_var("the-variable")) {
+ atf::utils::create_file(output, get_config_var("the-variable") +
+ std::string("\n"));
+ } else {
+ atf::utils::create_file(output, "NOT DEFINED\n");
+ }
+ }
+}
+ATF_TEST_CASE_BODY(get_variable)
+{
+ ATF_REQUIRE_EQ("value2", get_config_var("the-variable"));
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, get_variable);
+}
diff --git a/integration/helpers/dump_env.cpp b/integration/helpers/dump_env.cpp
new file mode 100644
index 000000000000..a2e8313a0062
--- /dev/null
+++ b/integration/helpers/dump_env.cpp
@@ -0,0 +1,74 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Dumps all environment variables.
+//
+// This helper program allows comparing the printed environment variables
+// to what 'kyua report --verbose' may output. It does so by sorting the
+// variables and allowing the caller to customize how the output looks
+// like (indentation for each line and for continuation lines).
+
+#include <cstdlib>
+#include <iostream>
+
+#include "utils/env.hpp"
+#include "utils/text/operations.ipp"
+
+namespace text = utils::text;
+
+
+int
+main(const int argc, const char* const* const argv)
+{
+ if (argc != 3) {
+ std::cerr << "Usage: dump_env <prefix> <continuation-prefix>\n";
+ return EXIT_FAILURE;
+ }
+ const char* prefix = argv[1];
+ const char* continuation_prefix = argv[2];
+
+ const std::map< std::string, std::string > env = utils::getallenv();
+ for (std::map< std::string, std::string >::const_iterator
+ iter = env.begin(); iter != env.end(); ++iter) {
+ const std::string& name = (*iter).first;
+ const std::vector< std::string > value = text::split(
+ (*iter).second, '\n');
+
+ if (value.empty()) {
+ std::cout << prefix << name << "=\n";
+ } else {
+ std::cout << prefix << name << '=' << value[0] << '\n';
+ for (std::vector< std::string >::size_type i = 1;
+ i < value.size(); ++i) {
+ std::cout << continuation_prefix << value[i] << '\n';
+ }
+ }
+ }
+
+ return EXIT_SUCCESS;
+}
diff --git a/integration/helpers/expect_all_pass.cpp b/integration/helpers/expect_all_pass.cpp
new file mode 100644
index 000000000000..a7df16e3a783
--- /dev/null
+++ b/integration/helpers/expect_all_pass.cpp
@@ -0,0 +1,92 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+extern "C" {
+#include <signal.h>
+#include <unistd.h>
+}
+
+#include <cstdlib>
+
+#include <atf-c++.hpp>
+
+#include "utils/test_utils.ipp"
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(die);
+ATF_TEST_CASE_BODY(die)
+{
+ expect_death("This is the reason for death");
+ utils::abort_without_coredump();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(exit);
+ATF_TEST_CASE_BODY(exit)
+{
+ expect_exit(12, "Exiting with correct code");
+ std::exit(12);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(failure);
+ATF_TEST_CASE_BODY(failure)
+{
+ expect_fail("Oh no");
+ fail("Forced failure");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(signal);
+ATF_TEST_CASE_BODY(signal)
+{
+ expect_signal(SIGTERM, "Exiting with correct signal");
+ ::kill(::getpid(), SIGTERM);
+}
+
+
+ATF_TEST_CASE(timeout);
+ATF_TEST_CASE_HEAD(timeout)
+{
+ set_md_var("timeout", "1");
+}
+ATF_TEST_CASE_BODY(timeout)
+{
+ expect_timeout("This times out");
+ ::sleep(10);
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, die);
+ ATF_ADD_TEST_CASE(tcs, exit);
+ ATF_ADD_TEST_CASE(tcs, failure);
+ ATF_ADD_TEST_CASE(tcs, signal);
+ ATF_ADD_TEST_CASE(tcs, timeout);
+}
diff --git a/integration/helpers/expect_some_fail.cpp b/integration/helpers/expect_some_fail.cpp
new file mode 100644
index 000000000000..da1a9f0ceb39
--- /dev/null
+++ b/integration/helpers/expect_some_fail.cpp
@@ -0,0 +1,94 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+extern "C" {
+#include <signal.h>
+#include <unistd.h>
+}
+
+#include <cstdlib>
+
+#include <atf-c++.hpp>
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(die);
+ATF_TEST_CASE_BODY(die)
+{
+ expect_death("Won't die");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(exit);
+ATF_TEST_CASE_BODY(exit)
+{
+ expect_exit(12, "Invalid exit code");
+ std::exit(34);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(failure);
+ATF_TEST_CASE_BODY(failure)
+{
+ expect_fail("Does not fail");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(pass);
+ATF_TEST_CASE_BODY(pass)
+{
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(signal);
+ATF_TEST_CASE_BODY(signal)
+{
+ expect_signal(SIGTERM, "Invalid signal");
+ ::kill(::getpid(), SIGKILL);
+}
+
+
+ATF_TEST_CASE(timeout);
+ATF_TEST_CASE_HEAD(timeout)
+{
+ set_md_var("timeout", "1");
+}
+ATF_TEST_CASE_BODY(timeout)
+{
+ expect_timeout("Does not time out");
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, die);
+ ATF_ADD_TEST_CASE(tcs, exit);
+ ATF_ADD_TEST_CASE(tcs, failure);
+ ATF_ADD_TEST_CASE(tcs, pass);
+ ATF_ADD_TEST_CASE(tcs, signal);
+ ATF_ADD_TEST_CASE(tcs, timeout);
+}
diff --git a/integration/helpers/interrupts.cpp b/integration/helpers/interrupts.cpp
new file mode 100644
index 000000000000..b6c5a948098c
--- /dev/null
+++ b/integration/helpers/interrupts.cpp
@@ -0,0 +1,62 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+extern "C" {
+#include <unistd.h>
+}
+
+#include <atf-c++.hpp>
+
+#include <fstream>
+
+
+ATF_TEST_CASE_WITH_CLEANUP(block_body);
+ATF_TEST_CASE_HEAD(block_body)
+{
+ set_md_var("require.config", "body-cookie cleanup-cookie");
+}
+ATF_TEST_CASE_BODY(block_body)
+{
+ const std::string cookie(get_config_var("body-cookie"));
+ std::ofstream output(cookie.c_str());
+ output.close();
+ for (;;)
+ ::pause();
+}
+ATF_TEST_CASE_CLEANUP(block_body)
+{
+ const std::string cookie(get_config_var("cleanup-cookie"));
+ std::ofstream output(cookie.c_str());
+ output.close();
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, block_body);
+}
diff --git a/integration/helpers/metadata.cpp b/integration/helpers/metadata.cpp
new file mode 100644
index 000000000000..8005d7d9b68d
--- /dev/null
+++ b/integration/helpers/metadata.cpp
@@ -0,0 +1,95 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <cstdlib>
+#include <iostream>
+
+#include <atf-c++.hpp>
+
+#include "utils/test_utils.ipp"
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(no_properties);
+ATF_TEST_CASE_BODY(no_properties)
+{
+}
+
+
+ATF_TEST_CASE(one_property);
+ATF_TEST_CASE_HEAD(one_property)
+{
+ set_md_var("descr", "Does nothing but has one metadata property");
+}
+ATF_TEST_CASE_BODY(one_property)
+{
+ utils::abort_without_coredump();
+}
+
+
+ATF_TEST_CASE(many_properties);
+ATF_TEST_CASE_HEAD(many_properties)
+{
+ set_md_var("descr", " A description with some padding");
+ set_md_var("require.arch", "some-architecture");
+ set_md_var("require.config", "var1 var2 var3");
+ set_md_var("require.files", "/my/file1 /some/other/file");
+ set_md_var("require.machine", "some-platform");
+ set_md_var("require.progs", "bin1 bin2 /nonexistent/bin3");
+ set_md_var("require.user", "root");
+ set_md_var("X-no-meaning", "I am a custom variable");
+}
+ATF_TEST_CASE_BODY(many_properties)
+{
+ utils::abort_without_coredump();
+}
+
+
+ATF_TEST_CASE_WITH_CLEANUP(with_cleanup);
+ATF_TEST_CASE_HEAD(with_cleanup)
+{
+ set_md_var("timeout", "250");
+}
+ATF_TEST_CASE_BODY(with_cleanup)
+{
+ std::cout << "Body message to stdout\n";
+ std::cerr << "Body message to stderr\n";
+}
+ATF_TEST_CASE_CLEANUP(with_cleanup)
+{
+ std::cout << "Cleanup message to stdout\n";
+ std::cerr << "Cleanup message to stderr\n";
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, no_properties);
+ ATF_ADD_TEST_CASE(tcs, one_property);
+ ATF_ADD_TEST_CASE(tcs, many_properties);
+ ATF_ADD_TEST_CASE(tcs, with_cleanup);
+}
diff --git a/integration/helpers/race.cpp b/integration/helpers/race.cpp
new file mode 100644
index 000000000000..39d4b04f3923
--- /dev/null
+++ b/integration/helpers/race.cpp
@@ -0,0 +1,99 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file integration/helpers/race.cpp
+/// Creates a file and reads it back, looking for races.
+///
+/// This program should fail with high chances if it is called multiple times at
+/// once with TEST_ENV_shared_file pointing to the same file.
+
+extern "C" {
+#include <sys/types.h>
+
+#include <unistd.h>
+}
+
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+
+#include "utils/format/macros.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/env.hpp"
+#include "utils/optional.ipp"
+#include "utils/stream.hpp"
+
+namespace fs = utils::fs;
+
+using utils::optional;
+
+
+/// Entry point to the helper test program.
+///
+/// \return EXIT_SUCCESS if no race is detected; EXIT_FAILURE otherwise.
+int
+main(void)
+{
+ const optional< std::string > shared_file = utils::getenv(
+ "TEST_ENV_shared_file");
+ if (!shared_file) {
+ std::cerr << "Environment variable TEST_ENV_shared_file not defined\n";
+ std::exit(EXIT_FAILURE);
+ }
+ const fs::path shared_path(shared_file.get());
+
+ if (fs::exists(shared_path)) {
+ std::cerr << "Shared file already exists; created by a concurrent "
+ "test?";
+ std::exit(EXIT_FAILURE);
+ }
+
+ const std::string contents = F("%s") % ::getpid();
+
+ std::ofstream output(shared_path.c_str());
+ if (!output) {
+ std::cerr << "Failed to create shared file; conflict with a concurrent "
+ "test?";
+ std::exit(EXIT_FAILURE);
+ }
+ output << contents;
+ output.close();
+
+ ::usleep(10000);
+
+ const std::string read_contents = utils::read_file(shared_path);
+ if (read_contents != contents) {
+ std::cerr << "Shared file contains unexpected contents; modified by a "
+ "concurrent test?";
+ std::exit(EXIT_FAILURE);
+ }
+
+ fs::unlink(shared_path);
+ std::exit(EXIT_SUCCESS);
+}
diff --git a/integration/helpers/simple_all_pass.cpp b/integration/helpers/simple_all_pass.cpp
new file mode 100644
index 000000000000..4e168b4cca5f
--- /dev/null
+++ b/integration/helpers/simple_all_pass.cpp
@@ -0,0 +1,55 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <iostream>
+
+#include <atf-c++.hpp>
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(pass);
+ATF_TEST_CASE_BODY(pass)
+{
+ std::cout << "This is the stdout of pass\n";
+ std::cerr << "This is the stderr of pass\n";
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(skip);
+ATF_TEST_CASE_BODY(skip)
+{
+ std::cout << "This is the stdout of skip\n";
+ std::cerr << "This is the stderr of skip\n";
+ skip("The reason for skipping is this");
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, pass);
+ ATF_ADD_TEST_CASE(tcs, skip);
+}
diff --git a/integration/helpers/simple_some_fail.cpp b/integration/helpers/simple_some_fail.cpp
new file mode 100644
index 000000000000..909ffb6e2ee1
--- /dev/null
+++ b/integration/helpers/simple_some_fail.cpp
@@ -0,0 +1,53 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <iostream>
+
+#include <atf-c++.hpp>
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(fail);
+ATF_TEST_CASE_BODY(fail)
+{
+ std::cout << "This is the stdout of fail\n";
+ std::cerr << "This is the stderr of fail\n";
+ fail("This fails on purpose");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(pass);
+ATF_TEST_CASE_BODY(pass)
+{
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, fail);
+ ATF_ADD_TEST_CASE(tcs, pass);
+}
diff --git a/integration/utils.sh b/integration/utils.sh
new file mode 100755
index 000000000000..99565a1c9857
--- /dev/null
+++ b/integration/utils.sh
@@ -0,0 +1,177 @@
+# Copyright 2011 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# Subcommand to strip out the durations and timestamps in a report.
+#
+# This is to make the reports deterministic and thus easily testable. The
+# time deltas are replaced by the fixed string S.UUU and the timestamps are
+# replaced by the fixed strings YYYYMMDD.HHMMSS.ssssss and
+# YYYY-MM-DDTHH:MM:SS.ssssssZ depending on their original format.
+#
+# This variable should be used as shown here:
+#
+# atf_check ... -x kyua report "| ${utils_strip_times}"
+#
+# Use the utils_install_times_wrapper function to create a 'kyua' wrapper
+# script that automatically does this.
+# CHECK_STYLE_DISABLE
+utils_strip_times='sed -E \
+ -e "s,( |\[|\")[0-9][0-9]*.[0-9][0-9][0-9](s]|s|\"),\1S.UUU\2,g" \
+ -e "s,[0-9]{8}-[0-9]{6}-[0-9]{6},YYYYMMDD-HHMMSS-ssssss,g" \
+ -e "s,[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z,YYYY-MM-DDTHH:MM:SS.ssssssZ,g"'
+# CHECK_STYLE_ENABLE
+
+
+# Same as utils_strip_times but avoids stripping timestamp-based report IDs.
+#
+# This is to make the reports deterministic and thus easily testable. The
+# time deltas are replaced by the fixed string S.UUU and the timestamps are
+# replaced by the fixed string YYYY-MM-DDTHH:MM:SS.ssssssZ.
+# CHECK_STYLE_DISABLE
+utils_strip_times_but_not_ids='sed -E \
+ -e "s,( |\[|\")[0-9][0-9]*.[0-9][0-9][0-9](s]|s|\"),\1S.UUU\2,g" \
+ -e "s,[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z,YYYY-MM-DDTHH:MM:SS.ssssssZ,g"'
+# CHECK_STYLE_ENABLE
+
+
+# Computes the results id for a test suite run.
+#
+# The computed path is "generic" in the sense that it does not include a
+# real timestamp: it only includes a placeholder. This function should be
+# used along the utils_strip_times function so that the timestamps of
+# the real results files are stripped out.
+#
+# \param path Optional path to use; if not given, use the cwd.
+utils_results_id() {
+ local test_suite_id="$(utils_test_suite_id "${@}")"
+ echo "${test_suite_id}.YYYYMMDD-HHMMSS-ssssss"
+}
+
+
+# Computes the results file for a test suite run.
+#
+# The computed path is "generic" in the sense that it does not include a
+# real timestamp: it only includes a placeholder. This function should be
+# used along the utils_strip_times function so that the timestampts of the
+# real results files are stripped out.
+#
+# \param path Optional path to use; if not given, use the cwd.
+utils_results_file() {
+ echo "${HOME}/.kyua/store/results.$(utils_results_id "${@}").db"
+}
+
+
+# Copies a helper binary from the source directory to the work directory.
+#
+# \param name The name of the binary to copy.
+# \param destination The target location for the binary; can be either
+# a directory name or a file name.
+utils_cp_helper() {
+ local name="${1}"; shift
+ local destination="${1}"; shift
+
+ ln -s "$(atf_get_srcdir)"/helpers/"${name}" "${destination}"
+}
+
+
+# Creates a 'kyua' binary in the path that strips timing data off the output.
+#
+# Call this on test cases that wish to replace timing data in the *stdout* of
+# Kyua with the deterministic strings. This is to be used by tests that
+# validate the 'test' and 'report' subcommands.
+utils_install_times_wrapper() {
+ [ ! -x kyua ] || return
+ cat >kyua <<EOF
+#! /bin/sh
+
+PATH=${PATH}
+
+kyua "\${@}" >kyua.tmpout
+result=\${?}
+cat kyua.tmpout | ${utils_strip_times}
+exit \${result}
+EOF
+ chmod +x kyua
+ PATH="$(pwd):${PATH}"
+}
+
+
+# Creates a 'kyua' binary in the path that makes the output of 'test' stable.
+#
+# Call this on test cases that wish to replace timing data with deterministic
+# strings and that need the result lines in the output to be sorted
+# lexicographically. The latter hides the indeterminism caused by parallel
+# execution so that the output can be verified. For these reasons, this is to
+# be used exclusively by tests that validate the 'test' subcommand.
+utils_install_stable_test_wrapper() {
+ [ ! -x kyua ] || return
+ cat >kyua <<EOF
+#! /bin/sh
+
+PATH=${PATH}
+
+kyua "\${@}" >kyua.tmpout
+result=\${?}
+cat kyua.tmpout | ${utils_strip_times} >kyua.tmpout2
+
+# Sort the test result lines but keep the rest intact.
+grep '[^ ]*:[^ ]*' kyua.tmpout2 | sort >kyua.tmpout3
+grep -v '[^ ]*:[^ ]*' kyua.tmpout2 >kyua.tmpout4
+cat kyua.tmpout3 kyua.tmpout4
+
+exit \${result}
+EOF
+ chmod +x kyua
+ PATH="$(pwd):${PATH}"
+}
+
+
+# Defines a test case with a default head.
+utils_test_case() {
+ local name="${1}"; shift
+
+ atf_test_case "${name}"
+ eval "${name}_head() {
+ atf_set require.progs kyua
+ }"
+}
+
+
+# Computes the test suite identifier for results files files.
+#
+# \param path Optional path to use; if not given, use the cwd.
+utils_test_suite_id() {
+ local path=
+ if [ ${#} -gt 0 ]; then
+ path="$(cd ${1} && pwd)"; shift
+ else
+ path="$(pwd)"
+ fi
+ echo "${path}" | sed -e 's,^/,,' -e 's,/,_,g'
+}