aboutsummaryrefslogtreecommitdiffstats
path: root/engine
diff options
context:
space:
mode:
authorBrooks Davis <brooks@FreeBSD.org>2020-03-17 16:56:50 +0000
committerBrooks Davis <brooks@FreeBSD.org>2020-03-17 16:56:50 +0000
commit08334c51dbb99d9ecd2bb86a2d94ed06da9e167a (patch)
treec43eb24d59bd5c963583a5190caef80fc8387322 /engine
downloadsrc-08334c51dbb99d9ecd2bb86a2d94ed06da9e167a.tar.gz
src-08334c51dbb99d9ecd2bb86a2d94ed06da9e167a.zip
Import the kyua testing framework for infrastructure softwarevendor/kyua/0.13-a685f91vendor/kyua
Imported at 0.13 plus assumulated changes to git hash a685f91. Obtained from: https://github.com/jmmv/kyua Sponsored by: DARPA
Notes
Notes: svn path=/vendor/kyua/dist/; revision=359042 svn path=/vendor/kyua/0.13-a685f91/; revision=359043; tag=vendor/kyua/0.13-a685f91
Diffstat (limited to 'engine')
-rw-r--r--engine/Kyuafile17
-rw-r--r--engine/Makefile.am.inc155
-rw-r--r--engine/atf.cpp242
-rw-r--r--engine/atf.hpp72
-rw-r--r--engine/atf_helpers.cpp414
-rw-r--r--engine/atf_list.cpp196
-rw-r--r--engine/atf_list.hpp51
-rw-r--r--engine/atf_list_test.cpp278
-rw-r--r--engine/atf_result.cpp642
-rw-r--r--engine/atf_result.hpp114
-rw-r--r--engine/atf_result_fwd.hpp43
-rw-r--r--engine/atf_result_test.cpp788
-rw-r--r--engine/atf_test.cpp450
-rw-r--r--engine/config.cpp254
-rw-r--r--engine/config.hpp65
-rw-r--r--engine/config_fwd.hpp43
-rw-r--r--engine/config_test.cpp203
-rw-r--r--engine/exceptions.cpp81
-rw-r--r--engine/exceptions.hpp75
-rw-r--r--engine/exceptions_test.cpp69
-rw-r--r--engine/filters.cpp389
-rw-r--r--engine/filters.hpp134
-rw-r--r--engine/filters_fwd.hpp45
-rw-r--r--engine/filters_test.cpp594
-rw-r--r--engine/kyuafile.cpp694
-rw-r--r--engine/kyuafile.hpp96
-rw-r--r--engine/kyuafile_fwd.hpp43
-rw-r--r--engine/kyuafile_test.cpp606
-rw-r--r--engine/plain.cpp143
-rw-r--r--engine/plain.hpp67
-rw-r--r--engine/plain_helpers.cpp238
-rw-r--r--engine/plain_test.cpp207
-rw-r--r--engine/requirements.cpp293
-rw-r--r--engine/requirements.hpp51
-rw-r--r--engine/requirements_test.cpp511
-rw-r--r--engine/scanner.cpp216
-rw-r--r--engine/scanner.hpp76
-rw-r--r--engine/scanner_fwd.hpp59
-rw-r--r--engine/scanner_test.cpp476
-rw-r--r--engine/scheduler.cpp1373
-rw-r--r--engine/scheduler.hpp282
-rw-r--r--engine/scheduler_fwd.hpp61
-rw-r--r--engine/scheduler_test.cpp1242
-rw-r--r--engine/tap.cpp191
-rw-r--r--engine/tap.hpp67
-rw-r--r--engine/tap_helpers.cpp202
-rw-r--r--engine/tap_parser.cpp438
-rw-r--r--engine/tap_parser.hpp99
-rw-r--r--engine/tap_parser_fwd.hpp50
-rw-r--r--engine/tap_parser_test.cpp465
-rw-r--r--engine/tap_test.cpp218
51 files changed, 13878 insertions, 0 deletions
diff --git a/engine/Kyuafile b/engine/Kyuafile
new file mode 100644
index 000000000000..1baa63bc9118
--- /dev/null
+++ b/engine/Kyuafile
@@ -0,0 +1,17 @@
+syntax(2)
+
+test_suite("kyua")
+
+atf_test_program{name="atf_test"}
+atf_test_program{name="atf_list_test"}
+atf_test_program{name="atf_result_test"}
+atf_test_program{name="config_test"}
+atf_test_program{name="exceptions_test"}
+atf_test_program{name="filters_test"}
+atf_test_program{name="kyuafile_test"}
+atf_test_program{name="plain_test"}
+atf_test_program{name="requirements_test"}
+atf_test_program{name="scanner_test"}
+atf_test_program{name="tap_test"}
+atf_test_program{name="tap_parser_test"}
+atf_test_program{name="scheduler_test"}
diff --git a/engine/Makefile.am.inc b/engine/Makefile.am.inc
new file mode 100644
index 000000000000..baa7fe0bb8a0
--- /dev/null
+++ b/engine/Makefile.am.inc
@@ -0,0 +1,155 @@
+# Copyright 2010 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ENGINE_CFLAGS = $(STORE_CFLAGS) $(MODEL_CFLAGS) $(UTILS_CFLAGS)
+ENGINE_LIBS = libengine.a $(STORE_LIBS) $(MODEL_LIBS) $(UTILS_LIBS)
+
+noinst_LIBRARIES += libengine.a
+libengine_a_CPPFLAGS = $(STORE_CFLAGS) $(UTILS_CFLAGS)
+libengine_a_SOURCES = engine/atf.cpp
+libengine_a_SOURCES += engine/atf.hpp
+libengine_a_SOURCES += engine/atf_list.cpp
+libengine_a_SOURCES += engine/atf_list.hpp
+libengine_a_SOURCES += engine/atf_result.cpp
+libengine_a_SOURCES += engine/atf_result.hpp
+libengine_a_SOURCES += engine/atf_result_fwd.hpp
+libengine_a_SOURCES += engine/config.cpp
+libengine_a_SOURCES += engine/config.hpp
+libengine_a_SOURCES += engine/config_fwd.hpp
+libengine_a_SOURCES += engine/exceptions.cpp
+libengine_a_SOURCES += engine/exceptions.hpp
+libengine_a_SOURCES += engine/filters.cpp
+libengine_a_SOURCES += engine/filters.hpp
+libengine_a_SOURCES += engine/filters_fwd.hpp
+libengine_a_SOURCES += engine/kyuafile.cpp
+libengine_a_SOURCES += engine/kyuafile.hpp
+libengine_a_SOURCES += engine/kyuafile_fwd.hpp
+libengine_a_SOURCES += engine/plain.cpp
+libengine_a_SOURCES += engine/plain.hpp
+libengine_a_SOURCES += engine/requirements.cpp
+libengine_a_SOURCES += engine/requirements.hpp
+libengine_a_SOURCES += engine/scanner.cpp
+libengine_a_SOURCES += engine/scanner.hpp
+libengine_a_SOURCES += engine/scanner_fwd.hpp
+libengine_a_SOURCES += engine/tap.cpp
+libengine_a_SOURCES += engine/tap.hpp
+libengine_a_SOURCES += engine/tap_parser.cpp
+libengine_a_SOURCES += engine/tap_parser.hpp
+libengine_a_SOURCES += engine/tap_parser_fwd.hpp
+libengine_a_SOURCES += engine/scheduler.cpp
+libengine_a_SOURCES += engine/scheduler.hpp
+libengine_a_SOURCES += engine/scheduler_fwd.hpp
+
+if WITH_ATF
+tests_enginedir = $(pkgtestsdir)/engine
+
+tests_engine_DATA = engine/Kyuafile
+EXTRA_DIST += $(tests_engine_DATA)
+
+tests_engine_PROGRAMS = engine/atf_helpers
+engine_atf_helpers_SOURCES = engine/atf_helpers.cpp
+engine_atf_helpers_CXXFLAGS = $(UTILS_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_atf_helpers_LDADD = $(UTILS_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/atf_test
+engine_atf_test_SOURCES = engine/atf_test.cpp
+engine_atf_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_atf_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/atf_list_test
+engine_atf_list_test_SOURCES = engine/atf_list_test.cpp
+engine_atf_list_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_atf_list_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/atf_result_test
+engine_atf_result_test_SOURCES = engine/atf_result_test.cpp
+engine_atf_result_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_atf_result_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/config_test
+engine_config_test_SOURCES = engine/config_test.cpp
+engine_config_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_config_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/exceptions_test
+engine_exceptions_test_SOURCES = engine/exceptions_test.cpp
+engine_exceptions_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_exceptions_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/filters_test
+engine_filters_test_SOURCES = engine/filters_test.cpp
+engine_filters_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_filters_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/kyuafile_test
+engine_kyuafile_test_SOURCES = engine/kyuafile_test.cpp
+engine_kyuafile_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_kyuafile_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/plain_helpers
+engine_plain_helpers_SOURCES = engine/plain_helpers.cpp
+engine_plain_helpers_CXXFLAGS = $(UTILS_CFLAGS)
+engine_plain_helpers_LDADD = $(UTILS_LIBS)
+
+tests_engine_PROGRAMS += engine/plain_test
+engine_plain_test_SOURCES = engine/plain_test.cpp
+engine_plain_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_plain_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/requirements_test
+engine_requirements_test_SOURCES = engine/requirements_test.cpp
+engine_requirements_test_CXXFLAGS = $(ENGINE_CFLAGS) $(UTILS_TEST_CFLAGS) \
+ $(ATF_CXX_CFLAGS)
+engine_requirements_test_LDADD = $(ENGINE_LIBS) $(UTILS_TEST_LIBS) \
+ $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/scanner_test
+engine_scanner_test_SOURCES = engine/scanner_test.cpp
+engine_scanner_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_scanner_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/tap_helpers
+engine_tap_helpers_SOURCES = engine/tap_helpers.cpp
+engine_tap_helpers_CXXFLAGS = $(UTILS_CFLAGS)
+engine_tap_helpers_LDADD = $(UTILS_LIBS)
+
+tests_engine_PROGRAMS += engine/tap_test
+engine_tap_test_SOURCES = engine/tap_test.cpp
+engine_tap_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_tap_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/tap_parser_test
+engine_tap_parser_test_SOURCES = engine/tap_parser_test.cpp
+engine_tap_parser_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_tap_parser_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+
+tests_engine_PROGRAMS += engine/scheduler_test
+engine_scheduler_test_SOURCES = engine/scheduler_test.cpp
+engine_scheduler_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
+engine_scheduler_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
+endif
diff --git a/engine/atf.cpp b/engine/atf.cpp
new file mode 100644
index 000000000000..eb63be20b0e7
--- /dev/null
+++ b/engine/atf.cpp
@@ -0,0 +1,242 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/atf.hpp"
+
+extern "C" {
+#include <unistd.h>
+}
+
+#include <cerrno>
+#include <cstdlib>
+#include <fstream>
+
+#include "engine/atf_list.hpp"
+#include "engine/atf_result.hpp"
+#include "engine/exceptions.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "model/test_result.hpp"
+#include "utils/defs.hpp"
+#include "utils/env.hpp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/logging/macros.hpp"
+#include "utils/optional.ipp"
+#include "utils/process/exceptions.hpp"
+#include "utils/process/operations.hpp"
+#include "utils/process/status.hpp"
+#include "utils/stream.hpp"
+
+namespace config = utils::config;
+namespace fs = utils::fs;
+namespace process = utils::process;
+
+using utils::optional;
+
+
+namespace {
+
+
+/// Basename of the file containing the result written by the ATF test case.
+static const char* result_name = "result.atf";
+
+
+/// Magic numbers returned by exec_list when exec(2) fails.
+enum list_exit_code {
+ exit_eacces = 90,
+ exit_enoent,
+ exit_enoexec,
+};
+
+
+} // anonymous namespace
+
+
+/// Executes a test program's list operation.
+///
+/// This method is intended to be called within a subprocess and is expected
+/// to terminate execution either by exec(2)ing the test program or by
+/// exiting with a failure.
+///
+/// \param test_program The test program to execute.
+/// \param vars User-provided variables to pass to the test program.
+void
+engine::atf_interface::exec_list(const model::test_program& test_program,
+ const config::properties_map& vars) const
+{
+ utils::setenv("__RUNNING_INSIDE_ATF_RUN", "internal-yes-value");
+
+ process::args_vector args;
+ for (config::properties_map::const_iterator iter = vars.begin();
+ iter != vars.end(); ++iter) {
+ args.push_back(F("-v%s=%s") % (*iter).first % (*iter).second);
+ }
+
+ args.push_back("-l");
+ try {
+ process::exec_unsafe(test_program.absolute_path(), args);
+ } catch (const process::system_error& e) {
+ if (e.original_errno() == EACCES)
+ ::_exit(exit_eacces);
+ else if (e.original_errno() == ENOENT)
+ ::_exit(exit_enoent);
+ else if (e.original_errno() == ENOEXEC)
+ ::_exit(exit_enoexec);
+ throw;
+ }
+}
+
+
+/// Computes the test cases list of a test program.
+///
+/// \param status The termination status of the subprocess used to execute
+/// the exec_test() method or none if the test timed out.
+/// \param stdout_path Path to the file containing the stdout of the test.
+/// \param stderr_path Path to the file containing the stderr of the test.
+///
+/// \return A list of test cases.
+///
+/// \throw error If there is a problem parsing the test case list.
+model::test_cases_map
+engine::atf_interface::parse_list(const optional< process::status >& status,
+ const fs::path& stdout_path,
+ const fs::path& stderr_path) const
+{
+ const std::string stderr_contents = utils::read_file(stderr_path);
+ if (!stderr_contents.empty())
+ LW("Test case list wrote to stderr: " + stderr_contents);
+
+ if (!status)
+ throw engine::error("Test case list timed out");
+ if (status.get().exited()) {
+ const int exitstatus = status.get().exitstatus();
+ if (exitstatus == EXIT_SUCCESS) {
+ // Nothing to do; fall through.
+ } else if (exitstatus == exit_eacces) {
+ throw engine::error("Permission denied to run test program");
+ } else if (exitstatus == exit_enoent) {
+ throw engine::error("Cannot find test program");
+ } else if (exitstatus == exit_enoexec) {
+ throw engine::error("Invalid test program format");
+ } else {
+ throw engine::error("Test program did not exit cleanly");
+ }
+ } else {
+ throw engine::error("Test program received signal");
+ }
+
+ std::ifstream input(stdout_path.c_str());
+ if (!input)
+ throw engine::load_error(stdout_path, "Cannot open file for read");
+ const model::test_cases_map test_cases = parse_atf_list(input);
+
+ if (!stderr_contents.empty())
+ throw engine::error("Test case list wrote to stderr");
+
+ return test_cases;
+}
+
+
+/// Executes a test case of the test program.
+///
+/// This method is intended to be called within a subprocess and is expected
+/// to terminate execution either by exec(2)ing the test program or by
+/// exiting with a failure.
+///
+/// \param test_program The test program to execute.
+/// \param test_case_name Name of the test case to invoke.
+/// \param vars User-provided variables to pass to the test program.
+/// \param control_directory Directory where the interface may place control
+/// files.
+void
+engine::atf_interface::exec_test(const model::test_program& test_program,
+ const std::string& test_case_name,
+ const config::properties_map& vars,
+ const fs::path& control_directory) const
+{
+ utils::setenv("__RUNNING_INSIDE_ATF_RUN", "internal-yes-value");
+
+ process::args_vector args;
+ for (config::properties_map::const_iterator iter = vars.begin();
+ iter != vars.end(); ++iter) {
+ args.push_back(F("-v%s=%s") % (*iter).first % (*iter).second);
+ }
+
+ args.push_back(F("-r%s") % (control_directory / result_name));
+ args.push_back(test_case_name);
+ process::exec(test_program.absolute_path(), args);
+}
+
+
+/// Executes a test cleanup routine of the test program.
+///
+/// This method is intended to be called within a subprocess and is expected
+/// to terminate execution either by exec(2)ing the test program or by
+/// exiting with a failure.
+///
+/// \param test_program The test program to execute.
+/// \param test_case_name Name of the test case to invoke.
+/// \param vars User-provided variables to pass to the test program.
+void
+engine::atf_interface::exec_cleanup(
+ const model::test_program& test_program,
+ const std::string& test_case_name,
+ const config::properties_map& vars,
+ const fs::path& /* control_directory */) const
+{
+ utils::setenv("__RUNNING_INSIDE_ATF_RUN", "internal-yes-value");
+
+ process::args_vector args;
+ for (config::properties_map::const_iterator iter = vars.begin();
+ iter != vars.end(); ++iter) {
+ args.push_back(F("-v%s=%s") % (*iter).first % (*iter).second);
+ }
+
+ args.push_back(F("%s:cleanup") % test_case_name);
+ process::exec(test_program.absolute_path(), args);
+}
+
+
+/// Computes the result of a test case based on its termination status.
+///
+/// \param status The termination status of the subprocess used to execute
+/// the exec_test() method or none if the test timed out.
+/// \param control_directory Directory where the interface may have placed
+/// control files.
+///
+/// \return A test result.
+model::test_result
+engine::atf_interface::compute_result(
+ const optional< process::status >& status,
+ const fs::path& control_directory,
+ const fs::path& /* stdout_path */,
+ const fs::path& /* stderr_path */) const
+{
+ return calculate_atf_result(status, control_directory / result_name);
+}
diff --git a/engine/atf.hpp b/engine/atf.hpp
new file mode 100644
index 000000000000..34ddc2413235
--- /dev/null
+++ b/engine/atf.hpp
@@ -0,0 +1,72 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/atf.hpp
+/// Execution engine for test programs that implement the atf interface.
+
+#if !defined(ENGINE_ATF_HPP)
+#define ENGINE_ATF_HPP
+
+#include "engine/scheduler.hpp"
+
+namespace engine {
+
+
+/// Implementation of the scheduler interface for atf test programs.
+class atf_interface : public engine::scheduler::interface {
+public:
+ void exec_list(const model::test_program&,
+ const utils::config::properties_map&) const UTILS_NORETURN;
+
+ model::test_cases_map parse_list(
+ const utils::optional< utils::process::status >&,
+ const utils::fs::path&,
+ const utils::fs::path&) const;
+
+ void exec_test(const model::test_program&, const std::string&,
+ const utils::config::properties_map&,
+ const utils::fs::path&) const
+ UTILS_NORETURN;
+
+ void exec_cleanup(const model::test_program&, const std::string&,
+ const utils::config::properties_map&,
+ const utils::fs::path&) const
+ UTILS_NORETURN;
+
+ model::test_result compute_result(
+ const utils::optional< utils::process::status >&,
+ const utils::fs::path&,
+ const utils::fs::path&,
+ const utils::fs::path&) const;
+};
+
+
+} // namespace engine
+
+
+#endif // !defined(ENGINE_ATF_HPP)
diff --git a/engine/atf_helpers.cpp b/engine/atf_helpers.cpp
new file mode 100644
index 000000000000..c45654f10e58
--- /dev/null
+++ b/engine/atf_helpers.cpp
@@ -0,0 +1,414 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+extern "C" {
+#include <sys/stat.h>
+
+#include <signal.h>
+#include <unistd.h>
+}
+
+#include <algorithm>
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include <atf-c++.hpp>
+
+#include "utils/env.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/logging/operations.hpp"
+#include "utils/optional.ipp"
+#include "utils/test_utils.ipp"
+#include "utils/text/operations.ipp"
+
+namespace fs = utils::fs;
+namespace logging = utils::logging;
+namespace text = utils::text;
+
+using utils::optional;
+
+
+namespace {
+
+
+/// Creates an empty file in the given directory.
+///
+/// \param test_case The test case currently running.
+/// \param directory The name of the configuration variable that holds the path
+/// to the directory in which to create the cookie file.
+/// \param name The name of the cookie file to create.
+static void
+create_cookie(const atf::tests::tc* test_case, const char* directory,
+ const char* name)
+{
+ if (!test_case->has_config_var(directory))
+ test_case->fail(std::string(name) + " not provided");
+
+ const fs::path control_dir(test_case->get_config_var(directory));
+ std::ofstream file((control_dir / name).c_str());
+ if (!file)
+ test_case->fail("Failed to create the control cookie");
+ file.close();
+}
+
+
+} // anonymous namespace
+
+
+ATF_TEST_CASE_WITH_CLEANUP(check_cleanup_workdir);
+ATF_TEST_CASE_HEAD(check_cleanup_workdir)
+{
+ set_md_var("require.config", "control_dir");
+}
+ATF_TEST_CASE_BODY(check_cleanup_workdir)
+{
+ std::ofstream cookie("workdir_cookie");
+ cookie << "1234\n";
+ cookie.close();
+ skip("cookie created");
+}
+ATF_TEST_CASE_CLEANUP(check_cleanup_workdir)
+{
+ const fs::path control_dir(get_config_var("control_dir"));
+
+ std::ifstream cookie("workdir_cookie");
+ if (!cookie) {
+ std::ofstream((control_dir / "missing_cookie").c_str()).close();
+ std::exit(EXIT_FAILURE);
+ }
+
+ std::string value;
+ cookie >> value;
+ if (value != "1234") {
+ std::ofstream((control_dir / "invalid_cookie").c_str()).close();
+ std::exit(EXIT_FAILURE);
+ }
+
+ std::ofstream((control_dir / "cookie_ok").c_str()).close();
+ std::exit(EXIT_SUCCESS);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_configuration_variables);
+ATF_TEST_CASE_BODY(check_configuration_variables)
+{
+ ATF_REQUIRE(has_config_var("first"));
+ ATF_REQUIRE_EQ("some value", get_config_var("first"));
+
+ ATF_REQUIRE(has_config_var("second"));
+ ATF_REQUIRE_EQ("some other value", get_config_var("second"));
+}
+
+
+ATF_TEST_CASE(check_list_config);
+ATF_TEST_CASE_HEAD(check_list_config)
+{
+ std::string description = "Found:";
+
+ if (has_config_var("var1"))
+ description += " var1=" + get_config_var("var1");
+ if (has_config_var("var2"))
+ description += " var2=" + get_config_var("var2");
+
+ set_md_var("descr", description);
+}
+ATF_TEST_CASE_BODY(check_list_config)
+{
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_unprivileged);
+ATF_TEST_CASE_BODY(check_unprivileged)
+{
+ if (::getuid() == 0)
+ fail("Running as root, but I shouldn't be");
+
+ std::ofstream file("cookie");
+ if (!file)
+ fail("Failed to create the cookie; work directory probably owned by "
+ "root");
+ file.close();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(crash);
+ATF_TEST_CASE_BODY(crash)
+{
+ std::abort();
+}
+
+
+ATF_TEST_CASE(crash_head);
+ATF_TEST_CASE_HEAD(crash_head)
+{
+ utils::abort_without_coredump();
+}
+ATF_TEST_CASE_BODY(crash_head)
+{
+}
+
+
+ATF_TEST_CASE_WITH_CLEANUP(crash_cleanup);
+ATF_TEST_CASE_HEAD(crash_cleanup)
+{
+}
+ATF_TEST_CASE_BODY(crash_cleanup)
+{
+}
+ATF_TEST_CASE_CLEANUP(crash_cleanup)
+{
+ utils::abort_without_coredump();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(create_cookie_in_control_dir);
+ATF_TEST_CASE_BODY(create_cookie_in_control_dir)
+{
+ create_cookie(this, "control_dir", "cookie");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(create_cookie_in_workdir);
+ATF_TEST_CASE_BODY(create_cookie_in_workdir)
+{
+ std::ofstream file("cookie");
+ if (!file)
+ fail("Failed to create the cookie");
+ file.close();
+}
+
+
+ATF_TEST_CASE_WITH_CLEANUP(create_cookie_from_cleanup);
+ATF_TEST_CASE_HEAD(create_cookie_from_cleanup)
+{
+}
+ATF_TEST_CASE_BODY(create_cookie_from_cleanup)
+{
+}
+ATF_TEST_CASE_CLEANUP(create_cookie_from_cleanup)
+{
+ create_cookie(this, "control_dir", "cookie");
+}
+
+
+ATF_TEST_CASE_WITH_CLEANUP(expect_timeout);
+ATF_TEST_CASE_HEAD(expect_timeout)
+{
+ if (has_config_var("timeout"))
+ set_md_var("timeout", get_config_var("timeout"));
+}
+ATF_TEST_CASE_BODY(expect_timeout)
+{
+ expect_timeout("Times out on purpose");
+ ::sleep(10);
+ create_cookie(this, "control_dir", "cookie");
+}
+ATF_TEST_CASE_CLEANUP(expect_timeout)
+{
+ create_cookie(this, "control_dir", "cookie.cleanup");
+}
+
+
+ATF_TEST_CASE_WITH_CLEANUP(output);
+ATF_TEST_CASE_HEAD(output)
+{
+}
+ATF_TEST_CASE_BODY(output)
+{
+ std::cout << "Body message to stdout\n";
+ std::cerr << "Body message to stderr\n";
+}
+ATF_TEST_CASE_CLEANUP(output)
+{
+ std::cout << "Cleanup message to stdout\n";
+ std::cerr << "Cleanup message to stderr\n";
+}
+
+
+ATF_TEST_CASE(output_in_list);
+ATF_TEST_CASE_HEAD(output_in_list)
+{
+ std::cerr << "Should not write anything!\n";
+}
+ATF_TEST_CASE_BODY(output_in_list)
+{
+}
+
+
+ATF_TEST_CASE(pass);
+ATF_TEST_CASE_HEAD(pass)
+{
+ set_md_var("descr", "Always-passing test case");
+}
+ATF_TEST_CASE_BODY(pass)
+{
+}
+
+
+ATF_TEST_CASE_WITH_CLEANUP(shared_workdir);
+ATF_TEST_CASE_HEAD(shared_workdir)
+{
+}
+ATF_TEST_CASE_BODY(shared_workdir)
+{
+ atf::utils::create_file("shared_cookie", "");
+}
+ATF_TEST_CASE_CLEANUP(shared_workdir)
+{
+ if (!atf::utils::file_exists("shared_cookie"))
+ utils::abort_without_coredump();
+}
+
+
+ATF_TEST_CASE(spawn_blocking_child);
+ATF_TEST_CASE_HEAD(spawn_blocking_child)
+{
+ set_md_var("require.config", "control_dir");
+}
+ATF_TEST_CASE_BODY(spawn_blocking_child)
+{
+ pid_t pid = ::fork();
+ if (pid == -1)
+ fail("Cannot fork subprocess");
+ else if (pid == 0) {
+ for (;;)
+ ::pause();
+ } else {
+ const fs::path name = fs::path(get_config_var("control_dir")) / "pid";
+ std::ofstream pidfile(name.c_str());
+ ATF_REQUIRE(pidfile);
+ pidfile << pid;
+ pidfile.close();
+ }
+}
+
+
+ATF_TEST_CASE_WITH_CLEANUP(timeout_body);
+ATF_TEST_CASE_HEAD(timeout_body)
+{
+ if (has_config_var("timeout"))
+ set_md_var("timeout", get_config_var("timeout"));
+}
+ATF_TEST_CASE_BODY(timeout_body)
+{
+ ::sleep(10);
+ create_cookie(this, "control_dir", "cookie");
+}
+ATF_TEST_CASE_CLEANUP(timeout_body)
+{
+ create_cookie(this, "control_dir", "cookie.cleanup");
+}
+
+
+ATF_TEST_CASE_WITH_CLEANUP(timeout_cleanup);
+ATF_TEST_CASE_HEAD(timeout_cleanup)
+{
+}
+ATF_TEST_CASE_BODY(timeout_cleanup)
+{
+}
+ATF_TEST_CASE_CLEANUP(timeout_cleanup)
+{
+ ::sleep(10);
+ create_cookie(this, "control_dir", "cookie");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(validate_isolation);
+ATF_TEST_CASE_BODY(validate_isolation)
+{
+ ATF_REQUIRE(utils::getenv("HOME").get() != "fake-value");
+ ATF_REQUIRE(!utils::getenv("LANG"));
+}
+
+
+/// Wrapper around ATF_ADD_TEST_CASE to only add a test when requested.
+///
+/// The caller can set the TEST_CASES environment variable to a
+/// whitespace-separated list of test case names to enable. If not empty, the
+/// list acts as a filter for the tests to add.
+///
+/// \param tcs List of test cases into which to register the test.
+/// \param filters List of filters to determine whether the test applies or not.
+/// \param name Name of the test case being added.
+#define ADD_TEST_CASE(tcs, filters, name) \
+ do { \
+ if (filters.empty() || filters.find(#name) != filters.end()) \
+ ATF_ADD_TEST_CASE(tcs, name); \
+ } while (false)
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ logging::set_inmemory();
+
+ // TODO(jmmv): Instead of using "filters", we should make TEST_CASES
+ // explicitly list all the test cases to enable. This would let us get rid
+ // of some of the hacks below...
+ std::set< std::string > filters;
+
+ const optional< std::string > names_raw = utils::getenv("TEST_CASES");
+ if (names_raw) {
+ if (names_raw.get().empty())
+ return; // See TODO above.
+
+ const std::vector< std::string > names = text::split(
+ names_raw.get(), ' ');
+ std::copy(names.begin(), names.end(),
+ std::inserter(filters, filters.begin()));
+ }
+
+ if (filters.find("crash_head") != filters.end()) // See TODO above.
+ ATF_ADD_TEST_CASE(tcs, crash_head);
+ if (filters.find("output_in_list") != filters.end()) // See TODO above.
+ ATF_ADD_TEST_CASE(tcs, output_in_list);
+
+ ADD_TEST_CASE(tcs, filters, check_cleanup_workdir);
+ ADD_TEST_CASE(tcs, filters, check_configuration_variables);
+ ADD_TEST_CASE(tcs, filters, check_list_config);
+ ADD_TEST_CASE(tcs, filters, check_unprivileged);
+ ADD_TEST_CASE(tcs, filters, crash);
+ ADD_TEST_CASE(tcs, filters, crash_cleanup);
+ ADD_TEST_CASE(tcs, filters, create_cookie_in_control_dir);
+ ADD_TEST_CASE(tcs, filters, create_cookie_in_workdir);
+ ADD_TEST_CASE(tcs, filters, create_cookie_from_cleanup);
+ ADD_TEST_CASE(tcs, filters, expect_timeout);
+ ADD_TEST_CASE(tcs, filters, output);
+ ADD_TEST_CASE(tcs, filters, pass);
+ ADD_TEST_CASE(tcs, filters, shared_workdir);
+ ADD_TEST_CASE(tcs, filters, spawn_blocking_child);
+ ADD_TEST_CASE(tcs, filters, timeout_body);
+ ADD_TEST_CASE(tcs, filters, timeout_cleanup);
+ ADD_TEST_CASE(tcs, filters, validate_isolation);
+}
diff --git a/engine/atf_list.cpp b/engine/atf_list.cpp
new file mode 100644
index 000000000000..a16b889c74f0
--- /dev/null
+++ b/engine/atf_list.cpp
@@ -0,0 +1,196 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/atf_list.hpp"
+
+#include <fstream>
+#include <string>
+#include <utility>
+
+#include "engine/exceptions.hpp"
+#include "model/metadata.hpp"
+#include "model/test_case.hpp"
+#include "utils/config/exceptions.hpp"
+#include "utils/format/macros.hpp"
+
+namespace config = utils::config;
+namespace fs = utils::fs;
+
+
+namespace {
+
+
+/// Splits a property line of the form "name: word1 [... wordN]".
+///
+/// \param line The line to parse.
+///
+/// \return A (property_name, property_value) pair.
+///
+/// \throw format_error If the value of line is invalid.
+static std::pair< std::string, std::string >
+split_prop_line(const std::string& line)
+{
+ const std::string::size_type pos = line.find(": ");
+ if (pos == std::string::npos)
+ throw engine::format_error("Invalid property line; expecting line of "
+ "the form 'name: value'");
+ return std::make_pair(line.substr(0, pos), line.substr(pos + 2));
+}
+
+
+/// Parses a set of consecutive property lines.
+///
+/// Processing stops when an empty line or the end of file is reached. None of
+/// these conditions indicate errors.
+///
+/// \param input The stream to read the lines from.
+///
+/// \return The parsed property lines.
+///
+/// throw format_error If the input stream has an invalid format.
+static model::properties_map
+parse_properties(std::istream& input)
+{
+ model::properties_map properties;
+
+ std::string line;
+ while (std::getline(input, line).good() && !line.empty()) {
+ const std::pair< std::string, std::string > property = split_prop_line(
+ line);
+ if (properties.find(property.first) != properties.end())
+ throw engine::format_error("Duplicate value for property " +
+ property.first);
+ properties.insert(property);
+ }
+
+ return properties;
+}
+
+
+} // anonymous namespace
+
+
+/// Parses the metadata of an ATF test case.
+///
+/// \param props The properties (name/value string pairs) as provided by the
+/// ATF test program.
+///
+/// \return A parsed metadata object.
+///
+/// \throw engine::format_error If the syntax of any of the properties is
+/// invalid.
+model::metadata
+engine::parse_atf_metadata(const model::properties_map& props)
+{
+ model::metadata_builder mdbuilder;
+
+ try {
+ for (model::properties_map::const_iterator iter = props.begin();
+ iter != props.end(); iter++) {
+ const std::string& name = (*iter).first;
+ const std::string& value = (*iter).second;
+
+ if (name == "descr") {
+ mdbuilder.set_string("description", value);
+ } else if (name == "has.cleanup") {
+ mdbuilder.set_string("has_cleanup", value);
+ } else if (name == "require.arch") {
+ mdbuilder.set_string("allowed_architectures", value);
+ } else if (name == "require.config") {
+ mdbuilder.set_string("required_configs", value);
+ } else if (name == "require.files") {
+ mdbuilder.set_string("required_files", value);
+ } else if (name == "require.machine") {
+ mdbuilder.set_string("allowed_platforms", value);
+ } else if (name == "require.memory") {
+ mdbuilder.set_string("required_memory", value);
+ } else if (name == "require.progs") {
+ mdbuilder.set_string("required_programs", value);
+ } else if (name == "require.user") {
+ mdbuilder.set_string("required_user", value);
+ } else if (name == "timeout") {
+ mdbuilder.set_string("timeout", value);
+ } else if (name.length() > 2 && name.substr(0, 2) == "X-") {
+ mdbuilder.add_custom(name.substr(2), value);
+ } else {
+ throw engine::format_error(F("Unknown test case metadata "
+ "property '%s'") % name);
+ }
+ }
+ } catch (const config::error& e) {
+ throw engine::format_error(e.what());
+ }
+
+ return mdbuilder.build();
+}
+
+
+/// Parses the ATF list of test cases from an open stream.
+///
+/// \param input The stream to read from.
+///
+/// \return The collection of parsed test cases.
+///
+/// \throw format_error If there is any problem in the input data.
+model::test_cases_map
+engine::parse_atf_list(std::istream& input)
+{
+ std::string line;
+
+ std::getline(input, line);
+ if (line != "Content-Type: application/X-atf-tp; version=\"1\""
+ || !input.good())
+ throw format_error(F("Invalid header for test case list; expecting "
+ "Content-Type for application/X-atf-tp version 1, "
+ "got '%s'") % line);
+
+ std::getline(input, line);
+ if (!line.empty() || !input.good())
+ throw format_error(F("Invalid header for test case list; expecting "
+ "a blank line, got '%s'") % line);
+
+ model::test_cases_map_builder test_cases_builder;
+ while (std::getline(input, line).good()) {
+ const std::pair< std::string, std::string > ident = split_prop_line(
+ line);
+ if (ident.first != "ident" or ident.second.empty())
+ throw format_error("Invalid test case definition; must be "
+ "preceeded by the identifier");
+
+ const model::properties_map props = parse_properties(input);
+ test_cases_builder.add(ident.second, parse_atf_metadata(props));
+ }
+ const model::test_cases_map test_cases = test_cases_builder.build();
+ if (test_cases.empty()) {
+ // The scheduler interface also checks for the presence of at least one
+ // test case. However, because the atf format itself requires one test
+ // case to be always present, we check for this condition here as well.
+ throw format_error("No test cases");
+ }
+ return test_cases;
+}
diff --git a/engine/atf_list.hpp b/engine/atf_list.hpp
new file mode 100644
index 000000000000..3d81d03e3bcf
--- /dev/null
+++ b/engine/atf_list.hpp
@@ -0,0 +1,51 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/atf_list.hpp
+/// Parser of ATF test case lists.
+
+#if !defined(ENGINE_ATF_LIST_HPP)
+#define ENGINE_ATF_LIST_HPP
+
+#include <istream>
+
+#include "model/metadata_fwd.hpp"
+#include "model/test_case_fwd.hpp"
+#include "model/types.hpp"
+#include "utils/fs/path_fwd.hpp"
+
+namespace engine {
+
+
+model::metadata parse_atf_metadata(const model::properties_map&);
+model::test_cases_map parse_atf_list(std::istream&);
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_ATF_LIST_HPP)
diff --git a/engine/atf_list_test.cpp b/engine/atf_list_test.cpp
new file mode 100644
index 000000000000..7f19ca8fbec5
--- /dev/null
+++ b/engine/atf_list_test.cpp
@@ -0,0 +1,278 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/atf_list.hpp"
+
+#include <sstream>
+#include <string>
+
+#include <atf-c++.hpp>
+
+#include "engine/exceptions.hpp"
+#include "model/metadata.hpp"
+#include "model/test_case.hpp"
+#include "model/types.hpp"
+#include "utils/datetime.hpp"
+#include "utils/format/containers.ipp"
+#include "utils/fs/path.hpp"
+#include "utils/units.hpp"
+
+namespace datetime = utils::datetime;
+namespace fs = utils::fs;
+namespace units = utils::units;
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_metadata__defaults)
+ATF_TEST_CASE_BODY(parse_atf_metadata__defaults)
+{
+ const model::properties_map properties;
+ const model::metadata md = engine::parse_atf_metadata(properties);
+
+ const model::metadata exp_md = model::metadata_builder().build();
+ ATF_REQUIRE_EQ(exp_md, md);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_metadata__override_all)
+ATF_TEST_CASE_BODY(parse_atf_metadata__override_all)
+{
+ model::properties_map properties;
+ properties["descr"] = "Some text";
+ properties["has.cleanup"] = "true";
+ properties["require.arch"] = "i386 x86_64";
+ properties["require.config"] = "var1 var2 var3";
+ properties["require.files"] = "/file1 /dir/file2";
+ properties["require.machine"] = "amd64";
+ properties["require.memory"] = "1m";
+ properties["require.progs"] = "/bin/ls svn";
+ properties["require.user"] = "root";
+ properties["timeout"] = "123";
+ properties["X-foo"] = "value1";
+ properties["X-bar"] = "value2";
+ properties["X-baz-www"] = "value3";
+ const model::metadata md = engine::parse_atf_metadata(properties);
+
+ const model::metadata exp_md = model::metadata_builder()
+ .add_allowed_architecture("i386")
+ .add_allowed_architecture("x86_64")
+ .add_allowed_platform("amd64")
+ .add_custom("foo", "value1")
+ .add_custom("bar", "value2")
+ .add_custom("baz-www", "value3")
+ .add_required_config("var1")
+ .add_required_config("var2")
+ .add_required_config("var3")
+ .add_required_file(fs::path("/file1"))
+ .add_required_file(fs::path("/dir/file2"))
+ .add_required_program(fs::path("/bin/ls"))
+ .add_required_program(fs::path("svn"))
+ .set_description("Some text")
+ .set_has_cleanup(true)
+ .set_required_memory(units::bytes::parse("1m"))
+ .set_required_user("root")
+ .set_timeout(datetime::delta(123, 0))
+ .build();
+ ATF_REQUIRE_EQ(exp_md, md);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_metadata__unknown)
+ATF_TEST_CASE_BODY(parse_atf_metadata__unknown)
+{
+ model::properties_map properties;
+ properties["foobar"] = "Some text";
+
+ ATF_REQUIRE_THROW_RE(engine::format_error, "Unknown.*property.*'foobar'",
+ engine::parse_atf_metadata(properties));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__empty);
+ATF_TEST_CASE_BODY(parse_atf_list__empty)
+{
+ const std::string text = "";
+ std::istringstream input(text);
+ ATF_REQUIRE_THROW_RE(engine::format_error, "expecting Content-Type",
+ engine::parse_atf_list(input));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__invalid_header);
+ATF_TEST_CASE_BODY(parse_atf_list__invalid_header)
+{
+ {
+ const std::string text =
+ "Content-Type: application/X-atf-tp; version=\"1\"\n";
+ std::istringstream input(text);
+ ATF_REQUIRE_THROW_RE(engine::format_error, "expecting.*blank line",
+ engine::parse_atf_list(input));
+ }
+
+ {
+ const std::string text =
+ "Content-Type: application/X-atf-tp; version=\"1\"\nfoo\n";
+ std::istringstream input(text);
+ ATF_REQUIRE_THROW_RE(engine::format_error, "expecting.*blank line",
+ engine::parse_atf_list(input));
+ }
+
+ {
+ const std::string text =
+ "Content-Type: application/X-atf-tp; version=\"2\"\n\n";
+ std::istringstream input(text);
+ ATF_REQUIRE_THROW_RE(engine::format_error, "expecting Content-Type",
+ engine::parse_atf_list(input));
+ }
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__no_test_cases);
+ATF_TEST_CASE_BODY(parse_atf_list__no_test_cases)
+{
+ const std::string text =
+ "Content-Type: application/X-atf-tp; version=\"1\"\n\n";
+ std::istringstream input(text);
+ ATF_REQUIRE_THROW_RE(engine::format_error, "No test cases",
+ engine::parse_atf_list(input));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__one_test_case_simple);
+ATF_TEST_CASE_BODY(parse_atf_list__one_test_case_simple)
+{
+ const std::string text =
+ "Content-Type: application/X-atf-tp; version=\"1\"\n"
+ "\n"
+ "ident: test-case\n";
+ std::istringstream input(text);
+ const model::test_cases_map tests = engine::parse_atf_list(input);
+
+ const model::test_cases_map exp_tests = model::test_cases_map_builder()
+ .add("test-case").build();
+ ATF_REQUIRE_EQ(exp_tests, tests);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__one_test_case_complex);
+ATF_TEST_CASE_BODY(parse_atf_list__one_test_case_complex)
+{
+ const std::string text =
+ "Content-Type: application/X-atf-tp; version=\"1\"\n"
+ "\n"
+ "ident: first\n"
+ "descr: This is the description\n"
+ "timeout: 500\n";
+ std::istringstream input(text);
+ const model::test_cases_map tests = engine::parse_atf_list(input);
+
+ const model::test_cases_map exp_tests = model::test_cases_map_builder()
+ .add("first", model::metadata_builder()
+ .set_description("This is the description")
+ .set_timeout(datetime::delta(500, 0))
+ .build())
+ .build();
+ ATF_REQUIRE_EQ(exp_tests, tests);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__one_test_case_invalid_syntax);
+ATF_TEST_CASE_BODY(parse_atf_list__one_test_case_invalid_syntax)
+{
+ const std::string text =
+ "Content-Type: application/X-atf-tp; version=\"1\"\n\n"
+ "descr: This is the description\n"
+ "ident: first\n";
+ std::istringstream input(text);
+ ATF_REQUIRE_THROW_RE(engine::format_error, "preceeded.*identifier",
+ engine::parse_atf_list(input));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__one_test_case_invalid_properties);
+ATF_TEST_CASE_BODY(parse_atf_list__one_test_case_invalid_properties)
+{
+ // Inject a single invalid property that makes test_case::from_properties()
+ // raise a particular error message so that we can validate that such
+ // function was called. We do intensive testing separately, so it is not
+ // necessary to redo it here.
+ const std::string text =
+ "Content-Type: application/X-atf-tp; version=\"1\"\n\n"
+ "ident: first\n"
+ "require.progs: bin/ls\n";
+ std::istringstream input(text);
+ ATF_REQUIRE_THROW_RE(engine::format_error, "Relative path 'bin/ls'",
+ engine::parse_atf_list(input));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__many_test_cases);
+ATF_TEST_CASE_BODY(parse_atf_list__many_test_cases)
+{
+ const std::string text =
+ "Content-Type: application/X-atf-tp; version=\"1\"\n"
+ "\n"
+ "ident: first\n"
+ "descr: This is the description\n"
+ "\n"
+ "ident: second\n"
+ "timeout: 500\n"
+ "descr: Some text\n"
+ "\n"
+ "ident: third\n";
+ std::istringstream input(text);
+ const model::test_cases_map tests = engine::parse_atf_list(input);
+
+ const model::test_cases_map exp_tests = model::test_cases_map_builder()
+ .add("first", model::metadata_builder()
+ .set_description("This is the description")
+ .build())
+ .add("second", model::metadata_builder()
+ .set_description("Some text")
+ .set_timeout(datetime::delta(500, 0))
+ .build())
+ .add("third")
+ .build();
+ ATF_REQUIRE_EQ(exp_tests, tests);
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, parse_atf_metadata__defaults);
+ ATF_ADD_TEST_CASE(tcs, parse_atf_metadata__override_all);
+ ATF_ADD_TEST_CASE(tcs, parse_atf_metadata__unknown);
+
+ ATF_ADD_TEST_CASE(tcs, parse_atf_list__empty);
+ ATF_ADD_TEST_CASE(tcs, parse_atf_list__invalid_header);
+ ATF_ADD_TEST_CASE(tcs, parse_atf_list__no_test_cases);
+ ATF_ADD_TEST_CASE(tcs, parse_atf_list__one_test_case_simple);
+ ATF_ADD_TEST_CASE(tcs, parse_atf_list__one_test_case_complex);
+ ATF_ADD_TEST_CASE(tcs, parse_atf_list__one_test_case_invalid_syntax);
+ ATF_ADD_TEST_CASE(tcs, parse_atf_list__one_test_case_invalid_properties);
+ ATF_ADD_TEST_CASE(tcs, parse_atf_list__many_test_cases);
+}
diff --git a/engine/atf_result.cpp b/engine/atf_result.cpp
new file mode 100644
index 000000000000..f99b28f9e96e
--- /dev/null
+++ b/engine/atf_result.cpp
@@ -0,0 +1,642 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/atf_result.hpp"
+
+#include <cstdlib>
+#include <fstream>
+#include <utility>
+
+#include "engine/exceptions.hpp"
+#include "model/test_result.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/format/macros.hpp"
+#include "utils/optional.ipp"
+#include "utils/process/status.hpp"
+#include "utils/sanity.hpp"
+#include "utils/text/exceptions.hpp"
+#include "utils/text/operations.ipp"
+
+namespace fs = utils::fs;
+namespace process = utils::process;
+namespace text = utils::text;
+
+using utils::none;
+using utils::optional;
+
+
+namespace {
+
+
+/// Reads a file and flattens its lines.
+///
+/// The main purpose of this function is to simplify the parsing of a file
+/// containing the result of a test. Therefore, the return value carries
+/// several assumptions.
+///
+/// \param input The stream to read from.
+///
+/// \return A pair (line count, contents) detailing how many lines where read
+/// and their contents. If the file contains a single line with no newline
+/// character, the line count is 0. If the file includes more than one line,
+/// the lines are merged together and separated by the magic string
+/// '&lt;&lt;NEWLINE&gt;&gt;'.
+static std::pair< size_t, std::string >
+read_lines(std::istream& input)
+{
+ std::pair< size_t, std::string > ret = std::make_pair(0, "");
+
+ do {
+ std::string line;
+ std::getline(input, line);
+ if (input.eof() && !line.empty()) {
+ if (ret.first == 0)
+ ret.second = line;
+ else {
+ ret.second += "<<NEWLINE>>" + line;
+ ret.first++;
+ }
+ } else if (input.good()) {
+ if (ret.first == 0)
+ ret.second = line;
+ else
+ ret.second += "<<NEWLINE>>" + line;
+ ret.first++;
+ }
+ } while (input.good());
+
+ return ret;
+}
+
+
+/// Parses a test result that does not accept a reason.
+///
+/// \param status The result status name.
+/// \param rest The rest of the line after the status name.
+///
+/// \return An object representing the test result.
+///
+/// \throw format_error If the result is invalid (i.e. rest is invalid).
+///
+/// \pre status must be "passed".
+static engine::atf_result
+parse_without_reason(const std::string& status, const std::string& rest)
+{
+ if (!rest.empty())
+ throw engine::format_error(F("%s cannot have a reason") % status);
+ PRE(status == "passed");
+ return engine::atf_result(engine::atf_result::passed);
+}
+
+
+/// Parses a test result that needs a reason.
+///
+/// \param status The result status name.
+/// \param rest The rest of the line after the status name.
+///
+/// \return An object representing the test result.
+///
+/// \throw format_error If the result is invalid (i.e. rest is invalid).
+///
+/// \pre status must be one of "broken", "expected_death", "expected_failure",
+/// "expected_timeout", "failed" or "skipped".
+static engine::atf_result
+parse_with_reason(const std::string& status, const std::string& rest)
+{
+ using engine::atf_result;
+
+ if (rest.length() < 3 || rest.substr(0, 2) != ": ")
+ throw engine::format_error(F("%s must be followed by ': <reason>'") %
+ status);
+ const std::string reason = rest.substr(2);
+ INV(!reason.empty());
+
+ if (status == "broken")
+ return atf_result(atf_result::broken, reason);
+ else if (status == "expected_death")
+ return atf_result(atf_result::expected_death, reason);
+ else if (status == "expected_failure")
+ return atf_result(atf_result::expected_failure, reason);
+ else if (status == "expected_timeout")
+ return atf_result(atf_result::expected_timeout, reason);
+ else if (status == "failed")
+ return atf_result(atf_result::failed, reason);
+ else if (status == "skipped")
+ return atf_result(atf_result::skipped, reason);
+ else
+ PRE_MSG(false, "Unexpected status");
+}
+
+
+/// Converts a string to an integer.
+///
+/// \param str The string containing the integer to convert.
+///
+/// \return The converted integer; none if the parsing fails.
+static optional< int >
+parse_int(const std::string& str)
+{
+ try {
+ return utils::make_optional(text::to_type< int >(str));
+ } catch (const text::value_error& e) {
+ return none;
+ }
+}
+
+
+/// Parses a test result that needs a reason and accepts an optional integer.
+///
+/// \param status The result status name.
+/// \param rest The rest of the line after the status name.
+///
+/// \return The parsed test result if the data is valid, or a broken result if
+/// the parsing failed.
+///
+/// \pre status must be one of "expected_exit" or "expected_signal".
+static engine::atf_result
+parse_with_reason_and_arg(const std::string& status, const std::string& rest)
+{
+ using engine::atf_result;
+
+ std::string::size_type delim = rest.find_first_of(":(");
+ if (delim == std::string::npos)
+ throw engine::format_error(F("Invalid format for '%s' test case "
+ "result; must be followed by '[(num)]: "
+ "<reason>' but found '%s'") %
+ status % rest);
+
+ optional< int > arg;
+ if (rest[delim] == '(') {
+ const std::string::size_type delim2 = rest.find("):", delim);
+ if (delim == std::string::npos)
+ throw engine::format_error(F("Mismatched '(' in %s") % rest);
+
+ const std::string argstr = rest.substr(delim + 1, delim2 - delim - 1);
+ arg = parse_int(argstr);
+ if (!arg)
+ throw engine::format_error(F("Invalid integer argument '%s' to "
+ "'%s' test case result") %
+ argstr % status);
+ delim = delim2 + 1;
+ }
+
+ const std::string reason = rest.substr(delim + 2);
+
+ if (status == "expected_exit")
+ return atf_result(atf_result::expected_exit, arg, reason);
+ else if (status == "expected_signal")
+ return atf_result(atf_result::expected_signal, arg, reason);
+ else
+ PRE_MSG(false, "Unexpected status");
+}
+
+
+/// Formats the termination status of a process to be used with validate_result.
+///
+/// \param status The status to format.
+///
+/// \return A string describing the status.
+static std::string
+format_status(const process::status& status)
+{
+ if (status.exited())
+ return F("exited with code %s") % status.exitstatus();
+ else if (status.signaled())
+ return F("received signal %s%s") % status.termsig() %
+ (status.coredump() ? " (core dumped)" : "");
+ else
+ return F("terminated in an unknown manner");
+}
+
+
+} // anonymous namespace
+
+
+/// Constructs a raw result with a type.
+///
+/// The reason and the argument are left uninitialized.
+///
+/// \param type_ The type of the result.
+engine::atf_result::atf_result(const types type_) :
+ _type(type_)
+{
+}
+
+
+/// Constructs a raw result with a type and a reason.
+///
+/// The argument is left uninitialized.
+///
+/// \param type_ The type of the result.
+/// \param reason_ The reason for the result.
+engine::atf_result::atf_result(const types type_, const std::string& reason_) :
+ _type(type_), _reason(reason_)
+{
+}
+
+
+/// Constructs a raw result with a type, an optional argument and a reason.
+///
+/// \param type_ The type of the result.
+/// \param argument_ The optional argument for the result.
+/// \param reason_ The reason for the result.
+engine::atf_result::atf_result(const types type_,
+ const utils::optional< int >& argument_,
+ const std::string& reason_) :
+ _type(type_), _argument(argument_), _reason(reason_)
+{
+}
+
+
+/// Parses an input stream to extract a test result.
+///
+/// If the parsing fails for any reason, the test result is 'broken' and it
+/// contains the reason for the parsing failure. Test cases that report results
+/// in an inconsistent state cannot be trusted (e.g. the test program code may
+/// have a bug), and thus why they are reported as broken instead of just failed
+/// (which is a legitimate result for a test case).
+///
+/// \param input The stream to read from.
+///
+/// \return A generic representation of the result of the test case.
+///
+/// \throw format_error If the input is invalid.
+engine::atf_result
+engine::atf_result::parse(std::istream& input)
+{
+ const std::pair< size_t, std::string > data = read_lines(input);
+ if (data.first == 0)
+ throw format_error("Empty test result or no new line");
+ else if (data.first > 1)
+ throw format_error("Test result contains multiple lines: " +
+ data.second);
+ else {
+ const std::string::size_type delim = data.second.find_first_not_of(
+ "abcdefghijklmnopqrstuvwxyz_");
+ const std::string status = data.second.substr(0, delim);
+ const std::string rest = data.second.substr(status.length());
+
+ if (status == "broken")
+ return parse_with_reason(status, rest);
+ else if (status == "expected_death")
+ return parse_with_reason(status, rest);
+ else if (status == "expected_exit")
+ return parse_with_reason_and_arg(status, rest);
+ else if (status == "expected_failure")
+ return parse_with_reason(status, rest);
+ else if (status == "expected_signal")
+ return parse_with_reason_and_arg(status, rest);
+ else if (status == "expected_timeout")
+ return parse_with_reason(status, rest);
+ else if (status == "failed")
+ return parse_with_reason(status, rest);
+ else if (status == "passed")
+ return parse_without_reason(status, rest);
+ else if (status == "skipped")
+ return parse_with_reason(status, rest);
+ else
+ throw format_error(F("Unknown test result '%s'") % status);
+ }
+}
+
+
+/// Loads a test case result from a file.
+///
+/// \param file The file to parse.
+///
+/// \return The parsed test case result if all goes well.
+///
+/// \throw std::runtime_error If the file does not exist.
+/// \throw engine::format_error If the contents of the file are bogus.
+engine::atf_result
+engine::atf_result::load(const fs::path& file)
+{
+ std::ifstream input(file.c_str());
+ if (!input)
+ throw std::runtime_error("Cannot open results file");
+ else
+ return parse(input);
+}
+
+
+/// Gets the type of the result.
+///
+/// \return A result type.
+engine::atf_result::types
+engine::atf_result::type(void) const
+{
+ return _type;
+}
+
+
+/// Gets the optional argument of the result.
+///
+/// \return The argument of the result if present; none otherwise.
+const optional< int >&
+engine::atf_result::argument(void) const
+{
+ return _argument;
+}
+
+
+/// Gets the optional reason of the result.
+///
+/// \return The reason of the result if present; none otherwise.
+const optional< std::string >&
+engine::atf_result::reason(void) const
+{
+ return _reason;
+}
+
+
+/// Checks whether the result should be reported as good or not.
+///
+/// \return True if the result can be considered "good", false otherwise.
+bool
+engine::atf_result::good(void) const
+{
+ switch (_type) {
+ case atf_result::expected_death:
+ case atf_result::expected_exit:
+ case atf_result::expected_failure:
+ case atf_result::expected_signal:
+ case atf_result::expected_timeout:
+ case atf_result::passed:
+ case atf_result::skipped:
+ return true;
+
+ case atf_result::broken:
+ case atf_result::failed:
+ return false;
+
+ default:
+ UNREACHABLE;
+ }
+}
+
+
+/// Reinterprets a raw result based on the termination status of the test case.
+///
+/// This reinterpretation ensures that the termination conditions of the program
+/// match what is expected of the paticular result reported by the test program.
+/// If such conditions do not match, the test program is considered bogus and is
+/// thus reported as broken.
+///
+/// This is just a helper function for calculate_result(); the real result of
+/// the test case cannot be inferred from apply() only.
+///
+/// \param status The exit status of the test program, or none if the test
+/// program timed out.
+///
+/// \result The adjusted result. The original result is transformed into broken
+/// if the exit status of the program does not match our expectations.
+engine::atf_result
+engine::atf_result::apply(const optional< process::status >& status)
+ const
+{
+ if (!status) {
+ if (_type != atf_result::expected_timeout)
+ return atf_result(atf_result::broken, "Test case body timed out");
+ else
+ return *this;
+ }
+
+ INV(status);
+ switch (_type) {
+ case atf_result::broken:
+ return *this;
+
+ case atf_result::expected_death:
+ return *this;
+
+ case atf_result::expected_exit:
+ if (status.get().exited()) {
+ if (_argument) {
+ if (_argument.get() == status.get().exitstatus())
+ return *this;
+ else
+ return atf_result(
+ atf_result::failed,
+ F("Test case expected to exit with code %s but got "
+ "code %s") %
+ _argument.get() % status.get().exitstatus());
+ } else
+ return *this;
+ } else
+ return atf_result(atf_result::broken, "Expected clean exit but " +
+ format_status(status.get()));
+
+ case atf_result::expected_failure:
+ if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS)
+ return *this;
+ else
+ return atf_result(atf_result::broken, "Expected failure should "
+ "have reported success but " +
+ format_status(status.get()));
+
+ case atf_result::expected_signal:
+ if (status.get().signaled()) {
+ if (_argument) {
+ if (_argument.get() == status.get().termsig())
+ return *this;
+ else
+ return atf_result(
+ atf_result::failed,
+ F("Test case expected to receive signal %s but "
+ "got %s") %
+ _argument.get() % status.get().termsig());
+ } else
+ return *this;
+ } else
+ return atf_result(atf_result::broken, "Expected signal but " +
+ format_status(status.get()));
+
+ case atf_result::expected_timeout:
+ return atf_result(atf_result::broken, "Expected timeout but " +
+ format_status(status.get()));
+
+ case atf_result::failed:
+ if (status.get().exited() && status.get().exitstatus() == EXIT_FAILURE)
+ return *this;
+ else
+ return atf_result(atf_result::broken, "Failed test case should "
+ "have reported failure but " +
+ format_status(status.get()));
+
+ case atf_result::passed:
+ if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS)
+ return *this;
+ else
+ return atf_result(atf_result::broken, "Passed test case should "
+ "have reported success but " +
+ format_status(status.get()));
+
+ case atf_result::skipped:
+ if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS)
+ return *this;
+ else
+ return atf_result(atf_result::broken, "Skipped test case should "
+ "have reported success but " +
+ format_status(status.get()));
+ }
+
+ UNREACHABLE;
+}
+
+
+/// Converts an internal result to the interface-agnostic representation.
+///
+/// \return A generic result instance representing this result.
+model::test_result
+engine::atf_result::externalize(void) const
+{
+ switch (_type) {
+ case atf_result::broken:
+ return model::test_result(model::test_result_broken, _reason.get());
+
+ case atf_result::expected_death:
+ case atf_result::expected_exit:
+ case atf_result::expected_failure:
+ case atf_result::expected_signal:
+ case atf_result::expected_timeout:
+ return model::test_result(model::test_result_expected_failure,
+ _reason.get());
+
+ case atf_result::failed:
+ return model::test_result(model::test_result_failed, _reason.get());
+
+ case atf_result::passed:
+ return model::test_result(model::test_result_passed);
+
+ case atf_result::skipped:
+ return model::test_result(model::test_result_skipped, _reason.get());
+
+ default:
+ UNREACHABLE;
+ }
+}
+
+
+/// Compares two raw results for equality.
+///
+/// \param other The result to compare to.
+///
+/// \return True if the two raw results are equal; false otherwise.
+bool
+engine::atf_result::operator==(const atf_result& other) const
+{
+ return _type == other._type && _argument == other._argument &&
+ _reason == other._reason;
+}
+
+
+/// Compares two raw results for inequality.
+///
+/// \param other The result to compare to.
+///
+/// \return True if the two raw results are different; false otherwise.
+bool
+engine::atf_result::operator!=(const atf_result& other) const
+{
+ return !(*this == other);
+}
+
+
+/// Injects the object into a stream.
+///
+/// \param output The stream into which to inject the object.
+/// \param object The object to format.
+///
+/// \return The output stream.
+std::ostream&
+engine::operator<<(std::ostream& output, const atf_result& object)
+{
+ std::string result_name;
+ switch (object.type()) {
+ case atf_result::broken: result_name = "broken"; break;
+ case atf_result::expected_death: result_name = "expected_death"; break;
+ case atf_result::expected_exit: result_name = "expected_exit"; break;
+ case atf_result::expected_failure: result_name = "expected_failure"; break;
+ case atf_result::expected_signal: result_name = "expected_signal"; break;
+ case atf_result::expected_timeout: result_name = "expected_timeout"; break;
+ case atf_result::failed: result_name = "failed"; break;
+ case atf_result::passed: result_name = "passed"; break;
+ case atf_result::skipped: result_name = "skipped"; break;
+ }
+
+ const optional< int >& argument = object.argument();
+
+ const optional< std::string >& reason = object.reason();
+
+ output << F("model::test_result{type=%s, argument=%s, reason=%s}")
+ % text::quote(result_name, '\'')
+ % (argument ? (F("%s") % argument.get()).str() : "none")
+ % (reason ? text::quote(reason.get(), '\'') : "none");
+
+ return output;
+}
+
+
+/// Calculates the user-visible result of a test case.
+///
+/// This function needs to perform magic to ensure that what the test case
+/// reports as its result is what the user should really see: i.e. it adjusts
+/// the reported status of the test to the exit conditions of its body and
+/// cleanup parts.
+///
+/// \param body_status The termination status of the process that executed
+/// the body of the test. None if the body timed out.
+/// \param results_file The path to the results file that the test case body is
+/// supposed to have created.
+///
+/// \return The calculated test case result.
+model::test_result
+engine::calculate_atf_result(const optional< process::status >& body_status,
+ const fs::path& results_file)
+{
+ using engine::atf_result;
+
+ atf_result result(atf_result::broken, "Unknown result");
+ try {
+ result = atf_result::load(results_file);
+ } catch (const engine::format_error& error) {
+ result = atf_result(atf_result::broken, error.what());
+ } catch (const std::runtime_error& error) {
+ if (body_status)
+ result = atf_result(
+ atf_result::broken, F("Premature exit; test case %s") %
+ format_status(body_status.get()));
+ else {
+ // The test case timed out. apply() handles this case later.
+ }
+ }
+
+ result = result.apply(body_status);
+
+ return result.externalize();
+}
diff --git a/engine/atf_result.hpp b/engine/atf_result.hpp
new file mode 100644
index 000000000000..55f8a117a237
--- /dev/null
+++ b/engine/atf_result.hpp
@@ -0,0 +1,114 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/atf_result.hpp
+/// Functions and types to process the results of ATF-based test cases.
+
+#if !defined(ENGINE_ATF_RESULT_HPP)
+#define ENGINE_ATF_RESULT_HPP
+
+#include "engine/atf_result_fwd.hpp"
+
+#include <istream>
+#include <ostream>
+
+#include "model/test_result_fwd.hpp"
+#include "utils/optional.hpp"
+#include "utils/fs/path_fwd.hpp"
+#include "utils/process/status_fwd.hpp"
+
+namespace engine {
+
+
+/// Internal representation of the raw result files of ATF-based tests.
+///
+/// This class is used exclusively to represent the transient result files read
+/// from test cases before generating the "public" version of the result. This
+/// class should actually not be exposed in the header files, but it is for
+/// testing purposes only.
+class atf_result {
+public:
+ /// List of possible types for the test case result.
+ enum types {
+ broken,
+ expected_death,
+ expected_exit,
+ expected_failure,
+ expected_signal,
+ expected_timeout,
+ failed,
+ passed,
+ skipped,
+ };
+
+private:
+ /// The test case result.
+ types _type;
+
+ /// The optional integral argument that may accompany the result.
+ ///
+ /// Should only be present if the type is expected_exit or expected_signal.
+ utils::optional< int > _argument;
+
+ /// A description of the test case result.
+ ///
+ /// Should always be present except for the passed type.
+ utils::optional< std::string > _reason;
+
+public:
+ atf_result(const types);
+ atf_result(const types, const std::string&);
+ atf_result(const types, const utils::optional< int >&, const std::string&);
+
+ static atf_result parse(std::istream&);
+ static atf_result load(const utils::fs::path&);
+
+ types type(void) const;
+ const utils::optional< int >& argument(void) const;
+ const utils::optional< std::string >& reason(void) const;
+
+ bool good(void) const;
+ atf_result apply(const utils::optional< utils::process::status >&) const;
+ model::test_result externalize(void) const;
+
+ bool operator==(const atf_result&) const;
+ bool operator!=(const atf_result&) const;
+};
+
+
+std::ostream& operator<<(std::ostream&, const atf_result&);
+
+
+model::test_result calculate_atf_result(
+ const utils::optional< utils::process::status >&,
+ const utils::fs::path&);
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_ATF_IFACE_RESULTS_HPP)
diff --git a/engine/atf_result_fwd.hpp b/engine/atf_result_fwd.hpp
new file mode 100644
index 000000000000..2a1440e4929c
--- /dev/null
+++ b/engine/atf_result_fwd.hpp
@@ -0,0 +1,43 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/atf_result_fwd.hpp
+/// Forward declarations for engine/atf_result.hpp
+
+#if !defined(ENGINE_ATF_RESULT_FWD_HPP)
+#define ENGINE_ATF_RESULT_FWD_HPP
+
+namespace engine {
+
+
+class atf_result;
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_ATF_RESULT_FWD_HPP)
diff --git a/engine/atf_result_test.cpp b/engine/atf_result_test.cpp
new file mode 100644
index 000000000000..8ec61dc3c07e
--- /dev/null
+++ b/engine/atf_result_test.cpp
@@ -0,0 +1,788 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/atf_result.hpp"
+
+extern "C" {
+#include <signal.h>
+}
+
+#include <cstdlib>
+#include <fstream>
+#include <sstream>
+#include <stdexcept>
+
+#include <atf-c++.hpp>
+
+#include "engine/exceptions.hpp"
+#include "model/test_result.hpp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/process/status.hpp"
+
+namespace fs = utils::fs;
+namespace process = utils::process;
+
+using utils::none;
+using utils::optional;
+
+
+namespace {
+
+
+/// Performs a test for results::parse() that should succeed.
+///
+/// \param exp_type The expected type of the result.
+/// \param exp_argument The expected argument in the result, if any.
+/// \param exp_reason The expected reason describing the result, if any.
+/// \param text The literal input to parse; can include multiple lines.
+static void
+parse_ok_test(const engine::atf_result::types& exp_type,
+ const optional< int >& exp_argument,
+ const char* exp_reason, const char* text)
+{
+ std::istringstream input(text);
+ const engine::atf_result actual = engine::atf_result::parse(input);
+ ATF_REQUIRE(exp_type == actual.type());
+ ATF_REQUIRE_EQ(exp_argument, actual.argument());
+ if (exp_reason != NULL) {
+ ATF_REQUIRE(actual.reason());
+ ATF_REQUIRE_EQ(exp_reason, actual.reason().get());
+ } else {
+ ATF_REQUIRE(!actual.reason());
+ }
+}
+
+
+/// Wrapper around parse_ok_test to define a test case.
+///
+/// \param name The name of the test case; will be prefixed with
+/// "atf_result__parse__".
+/// \param exp_type The expected type of the result.
+/// \param exp_argument The expected argument in the result, if any.
+/// \param exp_reason The expected reason describing the result, if any.
+/// \param input The literal input to parse.
+#define PARSE_OK(name, exp_type, exp_argument, exp_reason, input) \
+ ATF_TEST_CASE_WITHOUT_HEAD(atf_result__parse__ ## name); \
+ ATF_TEST_CASE_BODY(atf_result__parse__ ## name) \
+ { \
+ parse_ok_test(exp_type, exp_argument, exp_reason, input); \
+ }
+
+
+/// Performs a test for results::parse() that should fail.
+///
+/// \param reason_regexp The reason to match against the broken reason.
+/// \param text The literal input to parse; can include multiple lines.
+static void
+parse_broken_test(const char* reason_regexp, const char* text)
+{
+ std::istringstream input(text);
+ ATF_REQUIRE_THROW_RE(engine::format_error, reason_regexp,
+ engine::atf_result::parse(input));
+}
+
+
+/// Wrapper around parse_broken_test to define a test case.
+///
+/// \param name The name of the test case; will be prefixed with
+/// "atf_result__parse__".
+/// \param reason_regexp The reason to match against the broken reason.
+/// \param input The literal input to parse.
+#define PARSE_BROKEN(name, reason_regexp, input) \
+ ATF_TEST_CASE_WITHOUT_HEAD(atf_result__parse__ ## name); \
+ ATF_TEST_CASE_BODY(atf_result__parse__ ## name) \
+ { \
+ parse_broken_test(reason_regexp, input); \
+ }
+
+
+} // anonymous namespace
+
+
+PARSE_BROKEN(empty,
+ "Empty.*no new line",
+ "");
+PARSE_BROKEN(no_newline__unknown,
+ "Empty.*no new line",
+ "foo");
+PARSE_BROKEN(no_newline__known,
+ "Empty.*no new line",
+ "passed");
+PARSE_BROKEN(multiline__no_newline,
+ "multiple lines.*foo<<NEWLINE>>bar",
+ "failed: foo\nbar");
+PARSE_BROKEN(multiline__with_newline,
+ "multiple lines.*foo<<NEWLINE>>bar",
+ "failed: foo\nbar\n");
+PARSE_BROKEN(unknown_status__no_reason,
+ "Unknown.*result.*'cba'",
+ "cba\n");
+PARSE_BROKEN(unknown_status__with_reason,
+ "Unknown.*result.*'hgf'",
+ "hgf: foo\n");
+PARSE_BROKEN(missing_reason__no_delim,
+ "failed.*followed by.*reason",
+ "failed\n");
+PARSE_BROKEN(missing_reason__bad_delim,
+ "failed.*followed by.*reason",
+ "failed:\n");
+PARSE_BROKEN(missing_reason__empty,
+ "failed.*followed by.*reason",
+ "failed: \n");
+
+
+PARSE_OK(broken__ok,
+ engine::atf_result::broken, none, "a b c",
+ "broken: a b c\n");
+PARSE_OK(broken__blanks,
+ engine::atf_result::broken, none, " ",
+ "broken: \n");
+
+
+PARSE_OK(expected_death__ok,
+ engine::atf_result::expected_death, none, "a b c",
+ "expected_death: a b c\n");
+PARSE_OK(expected_death__blanks,
+ engine::atf_result::expected_death, none, " ",
+ "expected_death: \n");
+
+
+PARSE_OK(expected_exit__ok__any,
+ engine::atf_result::expected_exit, none, "any exit code",
+ "expected_exit: any exit code\n");
+PARSE_OK(expected_exit__ok__specific,
+ engine::atf_result::expected_exit, optional< int >(712),
+ "some known exit code",
+ "expected_exit(712): some known exit code\n");
+PARSE_BROKEN(expected_exit__bad_int,
+ "Invalid integer.*45a3",
+ "expected_exit(45a3): this is broken\n");
+
+
+PARSE_OK(expected_failure__ok,
+ engine::atf_result::expected_failure, none, "a b c",
+ "expected_failure: a b c\n");
+PARSE_OK(expected_failure__blanks,
+ engine::atf_result::expected_failure, none, " ",
+ "expected_failure: \n");
+
+
+PARSE_OK(expected_signal__ok__any,
+ engine::atf_result::expected_signal, none, "any signal code",
+ "expected_signal: any signal code\n");
+PARSE_OK(expected_signal__ok__specific,
+ engine::atf_result::expected_signal, optional< int >(712),
+ "some known signal code",
+ "expected_signal(712): some known signal code\n");
+PARSE_BROKEN(expected_signal__bad_int,
+ "Invalid integer.*45a3",
+ "expected_signal(45a3): this is broken\n");
+
+
+PARSE_OK(expected_timeout__ok,
+ engine::atf_result::expected_timeout, none, "a b c",
+ "expected_timeout: a b c\n");
+PARSE_OK(expected_timeout__blanks,
+ engine::atf_result::expected_timeout, none, " ",
+ "expected_timeout: \n");
+
+
+PARSE_OK(failed__ok,
+ engine::atf_result::failed, none, "a b c",
+ "failed: a b c\n");
+PARSE_OK(failed__blanks,
+ engine::atf_result::failed, none, " ",
+ "failed: \n");
+
+
+PARSE_OK(passed__ok,
+ engine::atf_result::passed, none, NULL,
+ "passed\n");
+PARSE_BROKEN(passed__reason,
+ "cannot have a reason",
+ "passed a b c\n");
+
+
+PARSE_OK(skipped__ok,
+ engine::atf_result::skipped, none, "a b c",
+ "skipped: a b c\n");
+PARSE_OK(skipped__blanks,
+ engine::atf_result::skipped, none, " ",
+ "skipped: \n");
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__ok);
+ATF_TEST_CASE_BODY(atf_result__load__ok)
+{
+ std::ofstream output("result.txt");
+ ATF_REQUIRE(output);
+ output << "skipped: a b c\n";
+ output.close();
+
+ const engine::atf_result result = engine::atf_result::load(
+ utils::fs::path("result.txt"));
+ ATF_REQUIRE(engine::atf_result::skipped == result.type());
+ ATF_REQUIRE(!result.argument());
+ ATF_REQUIRE(result.reason());
+ ATF_REQUIRE_EQ("a b c", result.reason().get());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__missing_file);
+ATF_TEST_CASE_BODY(atf_result__load__missing_file)
+{
+ ATF_REQUIRE_THROW_RE(
+ std::runtime_error, "Cannot open",
+ engine::atf_result::load(utils::fs::path("result.txt")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__format_error);
+ATF_TEST_CASE_BODY(atf_result__load__format_error)
+{
+ std::ofstream output("abc.txt");
+ ATF_REQUIRE(output);
+ output << "passed: foo\n";
+ output.close();
+
+ ATF_REQUIRE_THROW_RE(engine::format_error, "cannot have a reason",
+ engine::atf_result::load(utils::fs::path("abc.txt")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__broken__ok);
+ATF_TEST_CASE_BODY(atf_result__apply__broken__ok)
+{
+ const engine::atf_result in_result(engine::atf_result::broken,
+ "Passthrough");
+ const process::status status = process::status::fake_exited(EXIT_SUCCESS);
+ ATF_REQUIRE_EQ(in_result, in_result.apply(utils::make_optional(status)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__timed_out);
+ATF_TEST_CASE_BODY(atf_result__apply__timed_out)
+{
+ const engine::atf_result timed_out(engine::atf_result::broken,
+ "Some arbitrary error");
+ ATF_REQUIRE_EQ(engine::atf_result(engine::atf_result::broken,
+ "Test case body timed out"),
+ timed_out.apply(none));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_death__ok);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_death__ok)
+{
+ const engine::atf_result in_result(engine::atf_result::expected_death,
+ "Passthrough");
+ const process::status status = process::status::fake_signaled(SIGINT, true);
+ ATF_REQUIRE_EQ(in_result, in_result.apply(utils::make_optional(status)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__ok);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__ok)
+{
+ const process::status success = process::status::fake_exited(EXIT_SUCCESS);
+ const process::status failure = process::status::fake_exited(EXIT_FAILURE);
+
+ const engine::atf_result any_code(engine::atf_result::expected_exit, none,
+ "The reason");
+ ATF_REQUIRE_EQ(any_code, any_code.apply(utils::make_optional(success)));
+ ATF_REQUIRE_EQ(any_code, any_code.apply(utils::make_optional(failure)));
+
+ const engine::atf_result a_code(engine::atf_result::expected_exit,
+ utils::make_optional(EXIT_FAILURE), "The reason");
+ ATF_REQUIRE_EQ(a_code, a_code.apply(utils::make_optional(failure)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__failed);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__failed)
+{
+ const process::status success = process::status::fake_exited(EXIT_SUCCESS);
+
+ const engine::atf_result a_code(engine::atf_result::expected_exit,
+ utils::make_optional(EXIT_FAILURE), "The reason");
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::failed,
+ "Test case expected to exit with code 1 but got "
+ "code 0"),
+ a_code.apply(utils::make_optional(success)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__broken);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__broken)
+{
+ const process::status sig3 = process::status::fake_signaled(3, false);
+
+ const engine::atf_result any_code(engine::atf_result::expected_exit, none,
+ "The reason");
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Expected clean exit but received signal 3"),
+ any_code.apply(utils::make_optional(sig3)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_failure__ok);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__ok)
+{
+ const process::status status = process::status::fake_exited(EXIT_SUCCESS);
+ const engine::atf_result xfailure(engine::atf_result::expected_failure,
+ "The reason");
+ ATF_REQUIRE_EQ(xfailure, xfailure.apply(utils::make_optional(status)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_failure__broken);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__broken)
+{
+ const process::status failure = process::status::fake_exited(EXIT_FAILURE);
+ const process::status sig3 = process::status::fake_signaled(3, true);
+ const process::status sig4 = process::status::fake_signaled(4, false);
+
+ const engine::atf_result xfailure(engine::atf_result::expected_failure,
+ "The reason");
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Expected failure should have reported success but "
+ "exited with code 1"),
+ xfailure.apply(utils::make_optional(failure)));
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Expected failure should have reported success but "
+ "received signal 3 (core dumped)"),
+ xfailure.apply(utils::make_optional(sig3)));
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Expected failure should have reported success but "
+ "received signal 4"),
+ xfailure.apply(utils::make_optional(sig4)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__ok);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__ok)
+{
+ const process::status sig1 = process::status::fake_signaled(1, false);
+ const process::status sig3 = process::status::fake_signaled(3, true);
+
+ const engine::atf_result any_sig(engine::atf_result::expected_signal, none,
+ "The reason");
+ ATF_REQUIRE_EQ(any_sig, any_sig.apply(utils::make_optional(sig1)));
+ ATF_REQUIRE_EQ(any_sig, any_sig.apply(utils::make_optional(sig3)));
+
+ const engine::atf_result a_sig(engine::atf_result::expected_signal,
+ utils::make_optional(3), "The reason");
+ ATF_REQUIRE_EQ(a_sig, a_sig.apply(utils::make_optional(sig3)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__failed);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__failed)
+{
+ const process::status sig5 = process::status::fake_signaled(5, false);
+
+ const engine::atf_result a_sig(engine::atf_result::expected_signal,
+ utils::make_optional(4), "The reason");
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::failed,
+ "Test case expected to receive signal 4 but got 5"),
+ a_sig.apply(utils::make_optional(sig5)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__broken);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__broken)
+{
+ const process::status success = process::status::fake_exited(EXIT_SUCCESS);
+
+ const engine::atf_result any_sig(engine::atf_result::expected_signal, none,
+ "The reason");
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Expected signal but exited with code 0"),
+ any_sig.apply(utils::make_optional(success)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_timeout__ok);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__ok)
+{
+ const engine::atf_result timeout(engine::atf_result::expected_timeout,
+ "The reason");
+ ATF_REQUIRE_EQ(timeout, timeout.apply(none));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_timeout__broken);
+ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__broken)
+{
+ const process::status status = process::status::fake_exited(EXIT_SUCCESS);
+ const engine::atf_result timeout(engine::atf_result::expected_timeout,
+ "The reason");
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Expected timeout but exited with code 0"),
+ timeout.apply(utils::make_optional(status)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__failed__ok);
+ATF_TEST_CASE_BODY(atf_result__apply__failed__ok)
+{
+ const process::status status = process::status::fake_exited(EXIT_FAILURE);
+ const engine::atf_result failed(engine::atf_result::failed, "The reason");
+ ATF_REQUIRE_EQ(failed, failed.apply(utils::make_optional(status)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__failed__broken);
+ATF_TEST_CASE_BODY(atf_result__apply__failed__broken)
+{
+ const process::status success = process::status::fake_exited(EXIT_SUCCESS);
+ const process::status sig3 = process::status::fake_signaled(3, true);
+ const process::status sig4 = process::status::fake_signaled(4, false);
+
+ const engine::atf_result failed(engine::atf_result::failed, "The reason");
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Failed test case should have reported failure but "
+ "exited with code 0"),
+ failed.apply(utils::make_optional(success)));
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Failed test case should have reported failure but "
+ "received signal 3 (core dumped)"),
+ failed.apply(utils::make_optional(sig3)));
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Failed test case should have reported failure but "
+ "received signal 4"),
+ failed.apply(utils::make_optional(sig4)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__passed__ok);
+ATF_TEST_CASE_BODY(atf_result__apply__passed__ok)
+{
+ const process::status status = process::status::fake_exited(EXIT_SUCCESS);
+ const engine::atf_result passed(engine::atf_result::passed);
+ ATF_REQUIRE_EQ(passed, passed.apply(utils::make_optional(status)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__passed__broken);
+ATF_TEST_CASE_BODY(atf_result__apply__passed__broken)
+{
+ const process::status failure = process::status::fake_exited(EXIT_FAILURE);
+ const process::status sig3 = process::status::fake_signaled(3, true);
+ const process::status sig4 = process::status::fake_signaled(4, false);
+
+ const engine::atf_result passed(engine::atf_result::passed);
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Passed test case should have reported success but "
+ "exited with code 1"),
+ passed.apply(utils::make_optional(failure)));
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Passed test case should have reported success but "
+ "received signal 3 (core dumped)"),
+ passed.apply(utils::make_optional(sig3)));
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Passed test case should have reported success but "
+ "received signal 4"),
+ passed.apply(utils::make_optional(sig4)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__skipped__ok);
+ATF_TEST_CASE_BODY(atf_result__apply__skipped__ok)
+{
+ const process::status status = process::status::fake_exited(EXIT_SUCCESS);
+ const engine::atf_result skipped(engine::atf_result::skipped, "The reason");
+ ATF_REQUIRE_EQ(skipped, skipped.apply(utils::make_optional(status)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__skipped__broken);
+ATF_TEST_CASE_BODY(atf_result__apply__skipped__broken)
+{
+ const process::status failure = process::status::fake_exited(EXIT_FAILURE);
+ const process::status sig3 = process::status::fake_signaled(3, true);
+ const process::status sig4 = process::status::fake_signaled(4, false);
+
+ const engine::atf_result skipped(engine::atf_result::skipped, "The reason");
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Skipped test case should have reported success but "
+ "exited with code 1"),
+ skipped.apply(utils::make_optional(failure)));
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Skipped test case should have reported success but "
+ "received signal 3 (core dumped)"),
+ skipped.apply(utils::make_optional(sig3)));
+ ATF_REQUIRE_EQ(
+ engine::atf_result(engine::atf_result::broken,
+ "Skipped test case should have reported success but "
+ "received signal 4"),
+ skipped.apply(utils::make_optional(sig4)));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__broken);
+ATF_TEST_CASE_BODY(atf_result__externalize__broken)
+{
+ const engine::atf_result raw(engine::atf_result::broken, "The reason");
+ const model::test_result expected(model::test_result_broken,
+ "The reason");
+ ATF_REQUIRE_EQ(expected, raw.externalize());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_death);
+ATF_TEST_CASE_BODY(atf_result__externalize__expected_death)
+{
+ const engine::atf_result raw(engine::atf_result::expected_death,
+ "The reason");
+ const model::test_result expected(model::test_result_expected_failure,
+ "The reason");
+ ATF_REQUIRE_EQ(expected, raw.externalize());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_exit);
+ATF_TEST_CASE_BODY(atf_result__externalize__expected_exit)
+{
+ const engine::atf_result raw(engine::atf_result::expected_exit,
+ "The reason");
+ const model::test_result expected(model::test_result_expected_failure,
+ "The reason");
+ ATF_REQUIRE_EQ(expected, raw.externalize());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_failure);
+ATF_TEST_CASE_BODY(atf_result__externalize__expected_failure)
+{
+ const engine::atf_result raw(engine::atf_result::expected_failure,
+ "The reason");
+ const model::test_result expected(model::test_result_expected_failure,
+ "The reason");
+ ATF_REQUIRE_EQ(expected, raw.externalize());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_signal);
+ATF_TEST_CASE_BODY(atf_result__externalize__expected_signal)
+{
+ const engine::atf_result raw(engine::atf_result::expected_signal,
+ "The reason");
+ const model::test_result expected(model::test_result_expected_failure,
+ "The reason");
+ ATF_REQUIRE_EQ(expected, raw.externalize());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_timeout);
+ATF_TEST_CASE_BODY(atf_result__externalize__expected_timeout)
+{
+ const engine::atf_result raw(engine::atf_result::expected_timeout,
+ "The reason");
+ const model::test_result expected(model::test_result_expected_failure,
+ "The reason");
+ ATF_REQUIRE_EQ(expected, raw.externalize());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__failed);
+ATF_TEST_CASE_BODY(atf_result__externalize__failed)
+{
+ const engine::atf_result raw(engine::atf_result::failed, "The reason");
+ const model::test_result expected(model::test_result_failed,
+ "The reason");
+ ATF_REQUIRE(expected == raw.externalize());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__passed);
+ATF_TEST_CASE_BODY(atf_result__externalize__passed)
+{
+ const engine::atf_result raw(engine::atf_result::passed);
+ const model::test_result expected(model::test_result_passed);
+ ATF_REQUIRE_EQ(expected, raw.externalize());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__skipped);
+ATF_TEST_CASE_BODY(atf_result__externalize__skipped)
+{
+ const engine::atf_result raw(engine::atf_result::skipped, "The reason");
+ const model::test_result expected(model::test_result_skipped,
+ "The reason");
+ ATF_REQUIRE_EQ(expected, raw.externalize());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__missing_file);
+ATF_TEST_CASE_BODY(calculate_atf_result__missing_file)
+{
+ using process::status;
+
+ const status body_status = status::fake_exited(EXIT_SUCCESS);
+ const model::test_result expected(
+ model::test_result_broken,
+ "Premature exit; test case exited with code 0");
+ ATF_REQUIRE_EQ(expected, engine::calculate_atf_result(
+ utils::make_optional(body_status), fs::path("foo")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__bad_file);
+ATF_TEST_CASE_BODY(calculate_atf_result__bad_file)
+{
+ using process::status;
+
+ const status body_status = status::fake_exited(EXIT_SUCCESS);
+ atf::utils::create_file("foo", "invalid\n");
+ const model::test_result expected(model::test_result_broken,
+ "Unknown test result 'invalid'");
+ ATF_REQUIRE_EQ(expected, engine::calculate_atf_result(
+ utils::make_optional(body_status), fs::path("foo")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__body_ok);
+ATF_TEST_CASE_BODY(calculate_atf_result__body_ok)
+{
+ using process::status;
+
+ atf::utils::create_file("result.txt", "skipped: Something\n");
+ const status body_status = status::fake_exited(EXIT_SUCCESS);
+ ATF_REQUIRE_EQ(
+ model::test_result(model::test_result_skipped, "Something"),
+ engine::calculate_atf_result(utils::make_optional(body_status),
+ fs::path("result.txt")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__body_bad);
+ATF_TEST_CASE_BODY(calculate_atf_result__body_bad)
+{
+ using process::status;
+
+ atf::utils::create_file("result.txt", "skipped: Something\n");
+ const status body_status = status::fake_exited(EXIT_FAILURE);
+ ATF_REQUIRE_EQ(
+ model::test_result(model::test_result_broken, "Skipped test case "
+ "should have reported success but exited with "
+ "code 1"),
+ engine::calculate_atf_result(utils::make_optional(body_status),
+ fs::path("result.txt")));
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__empty);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__no_newline__unknown);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__no_newline__known);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__multiline__no_newline);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__multiline__with_newline);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__unknown_status__no_reason);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__unknown_status__with_reason);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__no_delim);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__bad_delim);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__empty);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__broken__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__broken__blanks);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_death__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_death__blanks);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__ok__any);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__ok__specific);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__bad_int);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_failure__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_failure__blanks);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__ok__any);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__ok__specific);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__bad_int);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_timeout__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_timeout__blanks);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__failed__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__failed__blanks);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__passed__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__passed__reason);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__skipped__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__parse__skipped__blanks);
+
+ ATF_ADD_TEST_CASE(tcs, atf_result__load__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__load__missing_file);
+ ATF_ADD_TEST_CASE(tcs, atf_result__load__format_error);
+
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__broken__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__timed_out);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_death__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__failed);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__broken);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_failure__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_failure__broken);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__failed);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__broken);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_timeout__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_timeout__broken);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__failed__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__failed__broken);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__passed__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__passed__broken);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__skipped__ok);
+ ATF_ADD_TEST_CASE(tcs, atf_result__apply__skipped__broken);
+
+ ATF_ADD_TEST_CASE(tcs, atf_result__externalize__broken);
+ ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_death);
+ ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_exit);
+ ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_failure);
+ ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_signal);
+ ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_timeout);
+ ATF_ADD_TEST_CASE(tcs, atf_result__externalize__failed);
+ ATF_ADD_TEST_CASE(tcs, atf_result__externalize__passed);
+ ATF_ADD_TEST_CASE(tcs, atf_result__externalize__skipped);
+
+ ATF_ADD_TEST_CASE(tcs, calculate_atf_result__missing_file);
+ ATF_ADD_TEST_CASE(tcs, calculate_atf_result__bad_file);
+ ATF_ADD_TEST_CASE(tcs, calculate_atf_result__body_ok);
+ ATF_ADD_TEST_CASE(tcs, calculate_atf_result__body_bad);
+}
diff --git a/engine/atf_test.cpp b/engine/atf_test.cpp
new file mode 100644
index 000000000000..9fe7797f4362
--- /dev/null
+++ b/engine/atf_test.cpp
@@ -0,0 +1,450 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/atf.hpp"
+
+extern "C" {
+#include <sys/stat.h>
+
+#include <signal.h>
+}
+
+#include <atf-c++.hpp>
+
+#include "engine/config.hpp"
+#include "engine/scheduler.hpp"
+#include "model/metadata.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program_fwd.hpp"
+#include "model/test_result.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/datetime.hpp"
+#include "utils/env.hpp"
+#include "utils/format/containers.ipp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/optional.ipp"
+#include "utils/stacktrace.hpp"
+#include "utils/test_utils.ipp"
+
+namespace config = utils::config;
+namespace datetime = utils::datetime;
+namespace fs = utils::fs;
+namespace scheduler = engine::scheduler;
+
+using utils::none;
+
+
+namespace {
+
+
+/// Lists the test cases associated with an ATF test program.
+///
+/// \param program_name Basename of the test program to run.
+/// \param root Path to the base of the test suite.
+/// \param names_filter Whitespace-separated list of test cases that the helper
+/// test program is allowed to expose.
+/// \param user_config User-provided configuration.
+///
+/// \return The list of loaded test cases.
+static model::test_cases_map
+list_one(const char* program_name,
+ const fs::path& root,
+ const char* names_filter = NULL,
+ config::tree user_config = engine::empty_config())
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ const scheduler::lazy_test_program program(
+ "atf", fs::path(program_name), root, "the-suite",
+ model::metadata_builder().build(), user_config, handle);
+
+ if (names_filter != NULL)
+ utils::setenv("TEST_CASES", names_filter);
+ const model::test_cases_map test_cases = handle.list_tests(
+ &program, user_config);
+
+ handle.cleanup();
+
+ return test_cases;
+}
+
+
+/// Runs a bogus test program and checks the error result.
+///
+/// \param exp_error Expected error string to find.
+/// \param program_name Basename of the test program to run.
+/// \param root Path to the base of the test suite.
+/// \param names_filter Whitespace-separated list of test cases that the helper
+/// test program is allowed to expose.
+static void
+check_list_one_fail(const char* exp_error,
+ const char* program_name,
+ const fs::path& root,
+ const char* names_filter = NULL)
+{
+ const model::test_cases_map test_cases = list_one(
+ program_name, root, names_filter);
+
+ ATF_REQUIRE_EQ(1, test_cases.size());
+ const model::test_case& test_case = test_cases.begin()->second;
+ ATF_REQUIRE_EQ("__test_cases_list__", test_case.name());
+ ATF_REQUIRE(test_case.fake_result());
+ ATF_REQUIRE_MATCH(exp_error,
+ test_case.fake_result().get().reason());
+}
+
+
+/// Runs one ATF test program and checks its result.
+///
+/// \param tc Pointer to the calling test case, to obtain srcdir.
+/// \param test_case_name Name of the "test case" to select from the helper
+/// program.
+/// \param exp_result The expected result.
+/// \param user_config User-provided configuration.
+/// \param check_empty_output If true, verify that the output of the test is
+/// silent. This is just a hack to implement one of the test cases; we'd
+/// easily have a nicer abstraction here...
+static void
+run_one(const atf::tests::tc* tc, const char* test_case_name,
+ const model::test_result& exp_result,
+ config::tree user_config = engine::empty_config(),
+ const bool check_empty_output = false)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ const model::test_program_ptr program(new scheduler::lazy_test_program(
+ "atf", fs::path("atf_helpers"), fs::path(tc->get_config_var("srcdir")),
+ "the-suite", model::metadata_builder().build(),
+ user_config, handle));
+
+ (void)handle.spawn_test(program, test_case_name, user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: ");
+ atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: ");
+ ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result());
+ if (check_empty_output) {
+ ATF_REQUIRE(atf::utils::compare_file(result_handle->stdout_file().str(),
+ ""));
+ ATF_REQUIRE(atf::utils::compare_file(result_handle->stderr_file().str(),
+ ""));
+ }
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+} // anonymous namespace
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list__ok);
+ATF_TEST_CASE_BODY(list__ok)
+{
+ const model::test_cases_map test_cases = list_one(
+ "atf_helpers", fs::path(get_config_var("srcdir")), "pass crash");
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("crash")
+ .add("pass", model::metadata_builder()
+ .set_description("Always-passing test case")
+ .build())
+ .build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list__configuration_variables);
+ATF_TEST_CASE_BODY(list__configuration_variables)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.var1", "value1");
+ user_config.set_string("test_suites.the-suite.var2", "value2");
+
+ const model::test_cases_map test_cases = list_one(
+ "atf_helpers", fs::path(get_config_var("srcdir")), "check_list_config",
+ user_config);
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("check_list_config", model::metadata_builder()
+ .set_description("Found: var1=value1 var2=value2")
+ .build())
+ .build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list__current_directory);
+ATF_TEST_CASE_BODY(list__current_directory)
+{
+ const fs::path helpers = fs::path(get_config_var("srcdir")) / "atf_helpers";
+ ATF_REQUIRE(::symlink(helpers.c_str(), "atf_helpers") != -1);
+ const model::test_cases_map test_cases = list_one(
+ "atf_helpers", fs::path("."), "pass");
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("pass", model::metadata_builder()
+ .set_description("Always-passing test case")
+ .build())
+ .build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list__relative_path);
+ATF_TEST_CASE_BODY(list__relative_path)
+{
+ const fs::path helpers = fs::path(get_config_var("srcdir")) / "atf_helpers";
+ ATF_REQUIRE(::mkdir("dir1", 0755) != -1);
+ ATF_REQUIRE(::mkdir("dir1/dir2", 0755) != -1);
+ ATF_REQUIRE(::symlink(helpers.c_str(), "dir1/dir2/atf_helpers") != -1);
+ const model::test_cases_map test_cases = list_one(
+ "dir2/atf_helpers", fs::path("dir1"), "pass");
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("pass", model::metadata_builder()
+ .set_description("Always-passing test case")
+ .build())
+ .build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list__missing_test_program);
+ATF_TEST_CASE_BODY(list__missing_test_program)
+{
+ check_list_one_fail("Cannot find test program", "non-existent",
+ fs::current_path());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list__not_a_test_program);
+ATF_TEST_CASE_BODY(list__not_a_test_program)
+{
+ atf::utils::create_file("not-valid", "garbage\n");
+ ATF_REQUIRE(::chmod("not-valid", 0755) != -1);
+ check_list_one_fail("Invalid test program format", "not-valid",
+ fs::current_path());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list__no_permissions);
+ATF_TEST_CASE_BODY(list__no_permissions)
+{
+ atf::utils::create_file("not-executable", "garbage\n");
+ check_list_one_fail("Permission denied to run test program",
+ "not-executable", fs::current_path());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list__abort);
+ATF_TEST_CASE_BODY(list__abort)
+{
+ check_list_one_fail("Test program received signal", "atf_helpers",
+ fs::path(get_config_var("srcdir")),
+ "crash_head");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list__empty);
+ATF_TEST_CASE_BODY(list__empty)
+{
+ check_list_one_fail("No test cases", "atf_helpers",
+ fs::path(get_config_var("srcdir")),
+ "");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list__stderr_not_quiet);
+ATF_TEST_CASE_BODY(list__stderr_not_quiet)
+{
+ check_list_one_fail("Test case list wrote to stderr", "atf_helpers",
+ fs::path(get_config_var("srcdir")),
+ "output_in_list");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__passes);
+ATF_TEST_CASE_BODY(test__body_only__passes)
+{
+ const model::test_result exp_result(model::test_result_passed);
+ run_one(this, "pass", exp_result);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__crashes);
+ATF_TEST_CASE_BODY(test__body_only__crashes)
+{
+ utils::prepare_coredump_test(this);
+
+ const model::test_result exp_result(
+ model::test_result_broken,
+ F("Premature exit; test case received signal %s (core dumped)") %
+ SIGABRT);
+ run_one(this, "crash", exp_result);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__times_out);
+ATF_TEST_CASE_BODY(test__body_only__times_out)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.control_dir",
+ fs::current_path().str());
+ user_config.set_string("test_suites.the-suite.timeout", "1");
+
+ const model::test_result exp_result(
+ model::test_result_broken, "Test case body timed out");
+ run_one(this, "timeout_body", exp_result, user_config);
+
+ ATF_REQUIRE(!atf::utils::file_exists("cookie"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__configuration_variables);
+ATF_TEST_CASE_BODY(test__body_only__configuration_variables)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.first", "some value");
+ user_config.set_string("test_suites.the-suite.second", "some other value");
+
+ const model::test_result exp_result(model::test_result_passed);
+ run_one(this, "check_configuration_variables", exp_result, user_config);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__no_atf_run_warning);
+ATF_TEST_CASE_BODY(test__body_only__no_atf_run_warning)
+{
+ const model::test_result exp_result(model::test_result_passed);
+ run_one(this, "pass", exp_result, engine::empty_config(), true);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__body_times_out);
+ATF_TEST_CASE_BODY(test__body_and_cleanup__body_times_out)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.control_dir",
+ fs::current_path().str());
+ user_config.set_string("test_suites.the-suite.timeout", "1");
+
+ const model::test_result exp_result(
+ model::test_result_broken, "Test case body timed out");
+ run_one(this, "timeout_body", exp_result, user_config);
+
+ ATF_REQUIRE(!atf::utils::file_exists("cookie"));
+ ATF_REQUIRE(atf::utils::file_exists("cookie.cleanup"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__cleanup_crashes);
+ATF_TEST_CASE_BODY(test__body_and_cleanup__cleanup_crashes)
+{
+ const model::test_result exp_result(
+ model::test_result_broken,
+ "Test case cleanup did not terminate successfully");
+ run_one(this, "crash_cleanup", exp_result);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__cleanup_times_out);
+ATF_TEST_CASE_BODY(test__body_and_cleanup__cleanup_times_out)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.control_dir",
+ fs::current_path().str());
+
+ scheduler::cleanup_timeout = datetime::delta(1, 0);
+ const model::test_result exp_result(
+ model::test_result_broken, "Test case cleanup timed out");
+ run_one(this, "timeout_cleanup", exp_result, user_config);
+
+ ATF_REQUIRE(!atf::utils::file_exists("cookie"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__expect_timeout);
+ATF_TEST_CASE_BODY(test__body_and_cleanup__expect_timeout)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.control_dir",
+ fs::current_path().str());
+ user_config.set_string("test_suites.the-suite.timeout", "1");
+
+ const model::test_result exp_result(
+ model::test_result_expected_failure, "Times out on purpose");
+ run_one(this, "expect_timeout", exp_result, user_config);
+
+ ATF_REQUIRE(!atf::utils::file_exists("cookie"));
+ ATF_REQUIRE(atf::utils::file_exists("cookie.cleanup"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__shared_workdir);
+ATF_TEST_CASE_BODY(test__body_and_cleanup__shared_workdir)
+{
+ const model::test_result exp_result(model::test_result_passed);
+ run_one(this, "shared_workdir", exp_result);
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ scheduler::register_interface(
+ "atf", std::shared_ptr< scheduler::interface >(
+ new engine::atf_interface()));
+
+ ATF_ADD_TEST_CASE(tcs, list__ok);
+ ATF_ADD_TEST_CASE(tcs, list__configuration_variables);
+ ATF_ADD_TEST_CASE(tcs, list__current_directory);
+ ATF_ADD_TEST_CASE(tcs, list__relative_path);
+ ATF_ADD_TEST_CASE(tcs, list__missing_test_program);
+ ATF_ADD_TEST_CASE(tcs, list__not_a_test_program);
+ ATF_ADD_TEST_CASE(tcs, list__no_permissions);
+ ATF_ADD_TEST_CASE(tcs, list__abort);
+ ATF_ADD_TEST_CASE(tcs, list__empty);
+ ATF_ADD_TEST_CASE(tcs, list__stderr_not_quiet);
+
+ ATF_ADD_TEST_CASE(tcs, test__body_only__passes);
+ ATF_ADD_TEST_CASE(tcs, test__body_only__crashes);
+ ATF_ADD_TEST_CASE(tcs, test__body_only__times_out);
+ ATF_ADD_TEST_CASE(tcs, test__body_only__configuration_variables);
+ ATF_ADD_TEST_CASE(tcs, test__body_only__no_atf_run_warning);
+ ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__body_times_out);
+ ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__cleanup_crashes);
+ ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__cleanup_times_out);
+ ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__expect_timeout);
+ ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__shared_workdir);
+}
diff --git a/engine/config.cpp b/engine/config.cpp
new file mode 100644
index 000000000000..3f162a94fbb5
--- /dev/null
+++ b/engine/config.cpp
@@ -0,0 +1,254 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/config.hpp"
+
+#if defined(HAVE_CONFIG_H)
+# include "config.h"
+#endif
+
+#include <stdexcept>
+
+#include "engine/exceptions.hpp"
+#include "utils/config/exceptions.hpp"
+#include "utils/config/parser.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/passwd.hpp"
+#include "utils/text/exceptions.hpp"
+#include "utils/text/operations.ipp"
+
+namespace config = utils::config;
+namespace fs = utils::fs;
+namespace passwd = utils::passwd;
+namespace text = utils::text;
+
+
+namespace {
+
+
+/// Defines the schema of a configuration tree.
+///
+/// \param [in,out] tree The tree to populate. The tree should be empty on
+/// entry to prevent collisions with the keys defined in here.
+static void
+init_tree(config::tree& tree)
+{
+ tree.define< config::string_node >("architecture");
+ tree.define< config::positive_int_node >("parallelism");
+ tree.define< config::string_node >("platform");
+ tree.define< engine::user_node >("unprivileged_user");
+ tree.define_dynamic("test_suites");
+}
+
+
+/// Fills in a configuration tree with default values.
+///
+/// \param [in,out] tree The tree to populate. init_tree() must have been
+/// called on it beforehand.
+static void
+set_defaults(config::tree& tree)
+{
+ tree.set< config::string_node >("architecture", KYUA_ARCHITECTURE);
+ // TODO(jmmv): Automatically derive this from the number of CPUs in the
+ // machine and forcibly set to a value greater than 1. Still testing
+ // the new parallel implementation as of 2015-02-27 though.
+ tree.set< config::positive_int_node >("parallelism", 1);
+ tree.set< config::string_node >("platform", KYUA_PLATFORM);
+}
+
+
+/// Configuration parser specialization for Kyua configuration files.
+class config_parser : public config::parser {
+ /// Initializes the configuration tree.
+ ///
+ /// This is a callback executed when the configuration script invokes the
+ /// syntax() method. We populate the configuration tree from here with the
+ /// schema version requested by the file.
+ ///
+ /// \param [in,out] tree The tree to populate.
+ /// \param syntax_version The version of the file format as specified in the
+ /// configuration file.
+ ///
+ /// \throw config::syntax_error If the syntax_format/syntax_version
+ /// combination is not supported.
+ void
+ setup(config::tree& tree, const int syntax_version)
+ {
+ if (syntax_version < 1 || syntax_version > 2)
+ throw config::syntax_error(F("Unsupported config version %s") %
+ syntax_version);
+
+ init_tree(tree);
+ set_defaults(tree);
+ }
+
+public:
+ /// Initializes the parser.
+ ///
+ /// \param [out] tree_ The tree in which the results of the parsing will be
+ /// stored when parse() is called. Should be empty on entry. Because
+ /// we grab a reference to this object, the tree must remain valid for
+ /// the existence of the parser object.
+ explicit config_parser(config::tree& tree_) :
+ config::parser(tree_)
+ {
+ }
+};
+
+
+} // anonymous namespace
+
+
+/// Copies the node.
+///
+/// \return A dynamically-allocated node.
+config::detail::base_node*
+engine::user_node::deep_copy(void) const
+{
+ std::auto_ptr< user_node > new_node(new user_node());
+ new_node->_value = _value;
+ return new_node.release();
+}
+
+
+/// Pushes the node's value onto the Lua stack.
+///
+/// \param state The Lua state onto which to push the value.
+void
+engine::user_node::push_lua(lutok::state& state) const
+{
+ state.push_string(value().name);
+}
+
+
+/// Sets the value of the node from an entry in the Lua stack.
+///
+/// \param state The Lua state from which to get the value.
+/// \param value_index The stack index in which the value resides.
+///
+/// \throw value_error If the value in state(value_index) cannot be
+/// processed by this node.
+void
+engine::user_node::set_lua(lutok::state& state, const int value_index)
+{
+ if (state.is_number(value_index)) {
+ config::typed_leaf_node< passwd::user >::set(
+ passwd::find_user_by_uid(state.to_integer(-1)));
+ } else if (state.is_string(value_index)) {
+ config::typed_leaf_node< passwd::user >::set(
+ passwd::find_user_by_name(state.to_string(-1)));
+ } else
+ throw config::value_error("Invalid user identifier");
+}
+
+
+/// Sets the value of the node from a raw string representation.
+///
+/// \param raw_value The value to set the node to.
+///
+/// \throw value_error If the value is invalid.
+void
+engine::user_node::set_string(const std::string& raw_value)
+{
+ try {
+ config::typed_leaf_node< passwd::user >::set(
+ passwd::find_user_by_name(raw_value));
+ } catch (const std::runtime_error& e) {
+ int uid;
+ try {
+ uid = text::to_type< int >(raw_value);
+ } catch (const text::value_error& e2) {
+ throw error(F("Cannot find user with name '%s'") % raw_value);
+ }
+
+ try {
+ config::typed_leaf_node< passwd::user >::set(
+ passwd::find_user_by_uid(uid));
+ } catch (const std::runtime_error& e2) {
+ throw error(F("Cannot find user with UID %s") % uid);
+ }
+ }
+}
+
+
+/// Converts the contents of the node to a string.
+///
+/// \pre The node must have a value.
+///
+/// \return A string representation of the value held by the node.
+std::string
+engine::user_node::to_string(void) const
+{
+ return config::typed_leaf_node< passwd::user >::value().name;
+}
+
+
+/// Constructs a config with the built-in settings.
+///
+/// \return A default test suite configuration.
+config::tree
+engine::default_config(void)
+{
+ config::tree tree(false);
+ init_tree(tree);
+ set_defaults(tree);
+ return tree;
+}
+
+
+/// Constructs a config with the built-in settings.
+///
+/// \return An empty test suite configuration.
+config::tree
+engine::empty_config(void)
+{
+ config::tree tree(false);
+ init_tree(tree);
+ return tree;
+}
+
+
+/// Parses a test suite configuration file.
+///
+/// \param file The file to parse.
+///
+/// \return High-level representation of the configuration file.
+///
+/// \throw load_error If there is any problem loading the file. This includes
+/// file access errors and syntax errors.
+config::tree
+engine::load_config(const utils::fs::path& file)
+{
+ config::tree tree(false);
+ try {
+ config_parser(tree).parse(file);
+ } catch (const config::error& e) {
+ throw load_error(file, e.what());
+ }
+ return tree;
+}
diff --git a/engine/config.hpp b/engine/config.hpp
new file mode 100644
index 000000000000..2c1b83481862
--- /dev/null
+++ b/engine/config.hpp
@@ -0,0 +1,65 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/config.hpp
+/// Test suite configuration parsing and representation.
+
+#if !defined(ENGINE_CONFIG_HPP)
+#define ENGINE_CONFIG_HPP
+
+#include "engine/config_fwd.hpp"
+
+#include "utils/config/nodes.hpp"
+#include "utils/config/tree_fwd.hpp"
+#include "utils/fs/path_fwd.hpp"
+#include "utils/passwd_fwd.hpp"
+
+namespace engine {
+
+
+/// Tree node to hold a system user identifier.
+class user_node : public utils::config::typed_leaf_node< utils::passwd::user > {
+public:
+ virtual base_node* deep_copy(void) const;
+
+ void push_lua(lutok::state&) const;
+ void set_lua(lutok::state&, const int);
+
+ void set_string(const std::string&);
+ std::string to_string(void) const;
+};
+
+
+utils::config::tree default_config(void);
+utils::config::tree empty_config(void);
+utils::config::tree load_config(const utils::fs::path&);
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_CONFIG_HPP)
diff --git a/engine/config_fwd.hpp b/engine/config_fwd.hpp
new file mode 100644
index 000000000000..82da9b1382bd
--- /dev/null
+++ b/engine/config_fwd.hpp
@@ -0,0 +1,43 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/config_fwd.hpp
+/// Forward declarations for engine/config.hpp
+
+#if !defined(ENGINE_CONFIG_FWD_HPP)
+#define ENGINE_CONFIG_FWD_HPP
+
+namespace engine {
+
+
+class user_node;
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_CONFIG_FWD_HPP)
diff --git a/engine/config_test.cpp b/engine/config_test.cpp
new file mode 100644
index 000000000000..e4eb27421078
--- /dev/null
+++ b/engine/config_test.cpp
@@ -0,0 +1,203 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/config.hpp"
+
+#if defined(HAVE_CONFIG_H)
+# include "config.h"
+#endif
+
+#include <stdexcept>
+#include <vector>
+
+#include <atf-c++.hpp>
+
+#include "engine/exceptions.hpp"
+#include "utils/cmdline/exceptions.hpp"
+#include "utils/cmdline/parser.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/passwd.hpp"
+
+namespace config = utils::config;
+namespace fs = utils::fs;
+namespace passwd = utils::passwd;
+
+using utils::none;
+using utils::optional;
+
+
+namespace {
+
+
+/// Replaces the system user database with a fake one for testing purposes.
+static void
+set_mock_users(void)
+{
+ std::vector< passwd::user > users;
+ users.push_back(passwd::user("user1", 100, 150));
+ users.push_back(passwd::user("user2", 200, 250));
+ passwd::set_mock_users_for_testing(users);
+}
+
+
+/// Checks that the default values of a config object match our expectations.
+///
+/// This fails the test case if any field of the input config object is not
+/// what we expect.
+///
+/// \param config The configuration to validate.
+static void
+validate_defaults(const config::tree& config)
+{
+ ATF_REQUIRE_EQ(
+ KYUA_ARCHITECTURE,
+ config.lookup< config::string_node >("architecture"));
+
+ ATF_REQUIRE_EQ(
+ 1,
+ config.lookup< config::positive_int_node >("parallelism"));
+
+ ATF_REQUIRE_EQ(
+ KYUA_PLATFORM,
+ config.lookup< config::string_node >("platform"));
+
+ ATF_REQUIRE(!config.is_set("unprivileged_user"));
+
+ ATF_REQUIRE(config.all_properties("test_suites").empty());
+}
+
+
+} // anonymous namespace
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(config__defaults);
+ATF_TEST_CASE_BODY(config__defaults)
+{
+ const config::tree user_config = engine::default_config();
+ validate_defaults(user_config);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(config__set__parallelism);
+ATF_TEST_CASE_BODY(config__set__parallelism)
+{
+ config::tree user_config = engine::default_config();
+ user_config.set_string("parallelism", "8");
+ ATF_REQUIRE_THROW_RE(
+ config::error, "parallelism.*Must be a positive integer",
+ user_config.set_string("parallelism", "0"));
+ ATF_REQUIRE_THROW_RE(
+ config::error, "parallelism.*Must be a positive integer",
+ user_config.set_string("parallelism", "-1"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(config__load__defaults);
+ATF_TEST_CASE_BODY(config__load__defaults)
+{
+ atf::utils::create_file("config", "syntax(2)\n");
+
+ const config::tree user_config = engine::load_config(fs::path("config"));
+ validate_defaults(user_config);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(config__load__overrides);
+ATF_TEST_CASE_BODY(config__load__overrides)
+{
+ set_mock_users();
+
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "architecture = 'test-architecture'\n"
+ "parallelism = 16\n"
+ "platform = 'test-platform'\n"
+ "unprivileged_user = 'user2'\n"
+ "test_suites.mysuite.myvar = 'myvalue'\n");
+
+ const config::tree user_config = engine::load_config(fs::path("config"));
+
+ ATF_REQUIRE_EQ("test-architecture",
+ user_config.lookup_string("architecture"));
+ ATF_REQUIRE_EQ("16",
+ user_config.lookup_string("parallelism"));
+ ATF_REQUIRE_EQ("test-platform",
+ user_config.lookup_string("platform"));
+
+ const passwd::user& user = user_config.lookup< engine::user_node >(
+ "unprivileged_user");
+ ATF_REQUIRE_EQ("user2", user.name);
+ ATF_REQUIRE_EQ(200, user.uid);
+
+ config::properties_map exp_test_suites;
+ exp_test_suites["test_suites.mysuite.myvar"] = "myvalue";
+
+ ATF_REQUIRE(exp_test_suites == user_config.all_properties("test_suites"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(config__load__lua_error);
+ATF_TEST_CASE_BODY(config__load__lua_error)
+{
+ atf::utils::create_file("config", "this syntax is invalid\n");
+
+ ATF_REQUIRE_THROW(engine::load_error, engine::load_config(
+ fs::path("config")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(config__load__bad_syntax__version);
+ATF_TEST_CASE_BODY(config__load__bad_syntax__version)
+{
+ atf::utils::create_file("config", "syntax(123)\n");
+
+ ATF_REQUIRE_THROW_RE(engine::load_error,
+ "Unsupported config version 123",
+ engine::load_config(fs::path("config")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(config__load__missing_file);
+ATF_TEST_CASE_BODY(config__load__missing_file)
+{
+ ATF_REQUIRE_THROW_RE(engine::load_error, "Load of 'missing' failed",
+ engine::load_config(fs::path("missing")));
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, config__defaults);
+ ATF_ADD_TEST_CASE(tcs, config__set__parallelism);
+ ATF_ADD_TEST_CASE(tcs, config__load__defaults);
+ ATF_ADD_TEST_CASE(tcs, config__load__overrides);
+ ATF_ADD_TEST_CASE(tcs, config__load__lua_error);
+ ATF_ADD_TEST_CASE(tcs, config__load__bad_syntax__version);
+ ATF_ADD_TEST_CASE(tcs, config__load__missing_file);
+}
diff --git a/engine/exceptions.cpp b/engine/exceptions.cpp
new file mode 100644
index 000000000000..98a7b43a7de3
--- /dev/null
+++ b/engine/exceptions.cpp
@@ -0,0 +1,81 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/exceptions.hpp"
+
+#include "utils/format/macros.hpp"
+
+namespace fs = utils::fs;
+
+/// Constructs a new error with a plain-text message.
+///
+/// \param message The plain-text error message.
+engine::error::error(const std::string& message) :
+ std::runtime_error(message)
+{
+}
+
+
+/// Destructor for the error.
+engine::error::~error(void) throw()
+{
+}
+
+
+/// Constructs a new format_error.
+///
+/// \param reason_ Description of the format problem.
+engine::format_error::format_error(const std::string& reason_) :
+ error(reason_)
+{
+}
+
+
+/// Destructor for the error.
+engine::format_error::~format_error(void) throw()
+{
+}
+
+
+/// Constructs a new load_error.
+///
+/// \param file_ The file in which the error was encountered.
+/// \param reason_ Description of the load problem.
+engine::load_error::load_error(const fs::path& file_,
+ const std::string& reason_) :
+ error(F("Load of '%s' failed: %s") % file_ % reason_),
+ file(file_),
+ reason(reason_)
+{
+}
+
+
+/// Destructor for the error.
+engine::load_error::~load_error(void) throw()
+{
+}
diff --git a/engine/exceptions.hpp b/engine/exceptions.hpp
new file mode 100644
index 000000000000..fccb04f1aff2
--- /dev/null
+++ b/engine/exceptions.hpp
@@ -0,0 +1,75 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/exceptions.hpp
+/// Exception types raised by the engine module.
+
+#if !defined(ENGINE_EXCEPTIONS_HPP)
+#define ENGINE_EXCEPTIONS_HPP
+
+#include <stdexcept>
+
+#include "utils/fs/path.hpp"
+
+namespace engine {
+
+
+/// Base exception for engine errors.
+class error : public std::runtime_error {
+public:
+ explicit error(const std::string&);
+ virtual ~error(void) throw();
+};
+
+
+/// Error while processing data.
+class format_error : public error {
+public:
+ explicit format_error(const std::string&);
+ virtual ~format_error(void) throw();
+};
+
+
+/// Error while parsing external data.
+class load_error : public error {
+public:
+ /// The path to the file that caused the load error.
+ utils::fs::path file;
+
+ /// The reason for the error; may not include the file name.
+ std::string reason;
+
+ explicit load_error(const utils::fs::path&, const std::string&);
+ virtual ~load_error(void) throw();
+};
+
+
+} // namespace engine
+
+
+#endif // !defined(ENGINE_EXCEPTIONS_HPP)
diff --git a/engine/exceptions_test.cpp b/engine/exceptions_test.cpp
new file mode 100644
index 000000000000..16e7c9f33d16
--- /dev/null
+++ b/engine/exceptions_test.cpp
@@ -0,0 +1,69 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/exceptions.hpp"
+
+#include <cstring>
+
+#include <atf-c++.hpp>
+
+namespace fs = utils::fs;
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(error);
+ATF_TEST_CASE_BODY(error)
+{
+ const engine::error e("Some text");
+ ATF_REQUIRE(std::strcmp("Some text", e.what()) == 0);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(format_error);
+ATF_TEST_CASE_BODY(format_error)
+{
+ const engine::format_error e("Some text");
+ ATF_REQUIRE(std::strcmp("Some text", e.what()) == 0);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(load_error);
+ATF_TEST_CASE_BODY(load_error)
+{
+ const engine::load_error e(fs::path("/my/file"), "foo");
+ ATF_REQUIRE_EQ(fs::path("/my/file"), e.file);
+ ATF_REQUIRE_EQ("foo", e.reason);
+ ATF_REQUIRE(std::strcmp("Load of '/my/file' failed: foo", e.what()) == 0);
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, error);
+ ATF_ADD_TEST_CASE(tcs, format_error);
+ ATF_ADD_TEST_CASE(tcs, load_error);
+}
diff --git a/engine/filters.cpp b/engine/filters.cpp
new file mode 100644
index 000000000000..753e64ae05f8
--- /dev/null
+++ b/engine/filters.cpp
@@ -0,0 +1,389 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/filters.hpp"
+
+#include <algorithm>
+#include <stdexcept>
+
+#include "utils/format/macros.hpp"
+#include "utils/fs/exceptions.hpp"
+#include "utils/logging/macros.hpp"
+#include "utils/optional.ipp"
+#include "utils/sanity.hpp"
+
+namespace fs = utils::fs;
+
+using utils::none;
+using utils::optional;
+
+
+/// Constructs a filter.
+///
+/// \param test_program_ The name of the test program or of the subdirectory to
+/// match.
+/// \param test_case_ The name of the test case to match.
+engine::test_filter::test_filter(const fs::path& test_program_,
+ const std::string& test_case_) :
+ test_program(test_program_),
+ test_case(test_case_)
+{
+}
+
+
+/// Parses a user-provided test filter.
+///
+/// \param str The user-provided string representing a filter for tests. Must
+/// be of the form &lt;test_program%gt;[:&lt;test_case%gt;].
+///
+/// \return The parsed filter.
+///
+/// \throw std::runtime_error If the provided filter is invalid.
+engine::test_filter
+engine::test_filter::parse(const std::string& str)
+{
+ if (str.empty())
+ throw std::runtime_error("Test filter cannot be empty");
+
+ const std::string::size_type pos = str.find(':');
+ if (pos == 0)
+ throw std::runtime_error(F("Program name component in '%s' is empty")
+ % str);
+ if (pos == str.length() - 1)
+ throw std::runtime_error(F("Test case component in '%s' is empty")
+ % str);
+
+ try {
+ const fs::path test_program_(str.substr(0, pos));
+ if (test_program_.is_absolute())
+ throw std::runtime_error(F("Program name '%s' must be relative "
+ "to the test suite, not absolute") %
+ test_program_.str());
+ if (pos == std::string::npos) {
+ LD(F("Parsed user filter '%s': test program '%s', no test case") %
+ str % test_program_.str());
+ return test_filter(test_program_, "");
+ } else {
+ const std::string test_case_(str.substr(pos + 1));
+ LD(F("Parsed user filter '%s': test program '%s', test case '%s'") %
+ str % test_program_.str() % test_case_);
+ return test_filter(test_program_, test_case_);
+ }
+ } catch (const fs::error& e) {
+ throw std::runtime_error(F("Invalid path in filter '%s': %s") % str %
+ e.what());
+ }
+}
+
+
+/// Formats a filter for user presentation.
+///
+/// \return A user-friendly string representing the filter. Note that this does
+/// not necessarily match the string the user provided: in particular, the path
+/// may have been internally normalized.
+std::string
+engine::test_filter::str(void) const
+{
+ if (!test_case.empty())
+ return F("%s:%s") % test_program % test_case;
+ else
+ return test_program.str();
+}
+
+
+/// Checks if this filter contains another.
+///
+/// \param other The filter to compare to.
+///
+/// \return True if this filter contains the other filter or if they are equal.
+bool
+engine::test_filter::contains(const test_filter& other) const
+{
+ if (*this == other)
+ return true;
+ else
+ return test_case.empty() && test_program.is_parent_of(
+ other.test_program);
+}
+
+
+/// Checks if this filter matches a given test program name or subdirectory.
+///
+/// \param test_program_ The test program to compare to.
+///
+/// \return Whether the filter matches the test program. This is a superset of
+/// matches_test_case.
+bool
+engine::test_filter::matches_test_program(const fs::path& test_program_) const
+{
+ if (test_program == test_program_)
+ return true;
+ else {
+ // Check if the filter matches a subdirectory of the test program.
+ // The test case must be empty because we don't want foo:bar to match
+ // foo/baz.
+ return (test_case.empty() && test_program.is_parent_of(test_program_));
+ }
+}
+
+
+/// Checks if this filter matches a given test case identifier.
+///
+/// \param test_program_ The test program to compare to.
+/// \param test_case_ The test case to compare to.
+///
+/// \return Whether the filter matches the test case.
+bool
+engine::test_filter::matches_test_case(const fs::path& test_program_,
+ const std::string& test_case_) const
+{
+ if (matches_test_program(test_program_)) {
+ return test_case.empty() || test_case == test_case_;
+ } else
+ return false;
+}
+
+
+/// Less-than comparison for sorting purposes.
+///
+/// \param other The filter to compare to.
+///
+/// \return True if this filter sorts before the other filter.
+bool
+engine::test_filter::operator<(const test_filter& other) const
+{
+ return (
+ test_program < other.test_program ||
+ (test_program == other.test_program && test_case < other.test_case));
+}
+
+
+/// Equality comparison.
+///
+/// \param other The filter to compare to.
+///
+/// \return True if this filter is equal to the other filter.
+bool
+engine::test_filter::operator==(const test_filter& other) const
+{
+ return test_program == other.test_program && test_case == other.test_case;
+}
+
+
+/// Non-equality comparison.
+///
+/// \param other The filter to compare to.
+///
+/// \return True if this filter is different than the other filter.
+bool
+engine::test_filter::operator!=(const test_filter& other) const
+{
+ return !(*this == other);
+}
+
+
+/// Injects the object into a stream.
+///
+/// \param output The stream into which to inject the object.
+/// \param object The object to format.
+///
+/// \return The output stream.
+std::ostream&
+engine::operator<<(std::ostream& output, const test_filter& object)
+{
+ if (object.test_case.empty()) {
+ output << F("test_filter{test_program=%s}") % object.test_program;
+ } else {
+ output << F("test_filter{test_program=%s, test_case=%s}")
+ % object.test_program % object.test_case;
+ }
+ return output;
+}
+
+
+/// Constructs a new set of filters.
+///
+/// \param filters_ The filters themselves; if empty, no filters are applied.
+engine::test_filters::test_filters(const std::set< test_filter >& filters_) :
+ _filters(filters_)
+{
+}
+
+
+/// Checks if a given test program matches the set of filters.
+///
+/// This is provided as an optimization only, and the results of this function
+/// are less specific than those of match_test_case. Checking for the matching
+/// of a test program should be done before loading the list of test cases from
+/// a program, so as to avoid the delay in executing the test program, but
+/// match_test_case must still be called afterwards.
+///
+/// \param name The test program to check against the filters.
+///
+/// \return True if the provided identifier matches any filter.
+bool
+engine::test_filters::match_test_program(const fs::path& name) const
+{
+ if (_filters.empty())
+ return true;
+
+ bool matches = false;
+ for (std::set< test_filter >::const_iterator iter = _filters.begin();
+ !matches && iter != _filters.end(); iter++) {
+ matches = (*iter).matches_test_program(name);
+ }
+ return matches;
+}
+
+
+/// Checks if a given test case identifier matches the set of filters.
+///
+/// \param test_program The test program to check against the filters.
+/// \param test_case The test case to check against the filters.
+///
+/// \return A boolean indicating if the test case is matched by any filter and,
+/// if true, a string containing the filter name. The string is empty when
+/// there are no filters defined.
+engine::test_filters::match
+engine::test_filters::match_test_case(const fs::path& test_program,
+ const std::string& test_case) const
+{
+ if (_filters.empty()) {
+ INV(match_test_program(test_program));
+ return match(true, none);
+ }
+
+ optional< test_filter > found = none;
+ for (std::set< test_filter >::const_iterator iter = _filters.begin();
+ !found && iter != _filters.end(); iter++) {
+ if ((*iter).matches_test_case(test_program, test_case))
+ found = *iter;
+ }
+ INV(!found || match_test_program(test_program));
+ return match(static_cast< bool >(found), found);
+}
+
+
+/// Calculates the filters that have not matched any tests.
+///
+/// \param matched The filters that did match some tests. This must be a subset
+/// of the filters held by this object.
+///
+/// \return The set of filters that have not been used.
+std::set< engine::test_filter >
+engine::test_filters::difference(const std::set< test_filter >& matched) const
+{
+ PRE(std::includes(_filters.begin(), _filters.end(),
+ matched.begin(), matched.end()));
+
+ std::set< test_filter > filters;
+ std::set_difference(_filters.begin(), _filters.end(),
+ matched.begin(), matched.end(),
+ std::inserter(filters, filters.begin()));
+ return filters;
+}
+
+
+/// Checks if a collection of filters is disjoint.
+///
+/// \param filters The filters to check.
+///
+/// \throw std::runtime_error If the filters are not disjoint.
+void
+engine::check_disjoint_filters(const std::set< engine::test_filter >& filters)
+{
+ // Yes, this is an O(n^2) algorithm. However, we can assume that the number
+ // of test filters (which are provided by the user on the command line) on a
+ // particular run is in the order of tens, and thus this should not cause
+ // any serious performance trouble.
+ for (std::set< test_filter >::const_iterator i1 = filters.begin();
+ i1 != filters.end(); i1++) {
+ for (std::set< test_filter >::const_iterator i2 = filters.begin();
+ i2 != filters.end(); i2++) {
+ const test_filter& filter1 = *i1;
+ const test_filter& filter2 = *i2;
+
+ if (i1 != i2 && filter1.contains(filter2)) {
+ throw std::runtime_error(
+ F("Filters '%s' and '%s' are not disjoint") %
+ filter1.str() % filter2.str());
+ }
+ }
+ }
+}
+
+
+/// Constructs a filters_state instance.
+///
+/// \param filters_ The set of filters to track.
+engine::filters_state::filters_state(
+ const std::set< engine::test_filter >& filters_) :
+ _filters(test_filters(filters_))
+{
+}
+
+
+/// Checks whether these filters match the given test program.
+///
+/// \param test_program The test program to match against.
+///
+/// \return True if these filters match the given test program name.
+bool
+engine::filters_state::match_test_program(const fs::path& test_program) const
+{
+ return _filters.match_test_program(test_program);
+}
+
+
+/// Checks whether these filters match the given test case.
+///
+/// \param test_program The test program to match against.
+/// \param test_case The test case to match against.
+///
+/// \return True if these filters match the given test case identifier.
+bool
+engine::filters_state::match_test_case(const fs::path& test_program,
+ const std::string& test_case)
+{
+ engine::test_filters::match match = _filters.match_test_case(
+ test_program, test_case);
+ if (match.first && match.second)
+ _used_filters.insert(match.second.get());
+ return match.first;
+}
+
+
+/// Calculates the unused filters in this set.
+///
+/// \return Returns the set of filters that have not matched any tests. This
+/// information is useful to report usage errors to the user.
+std::set< engine::test_filter >
+engine::filters_state::unused(void) const
+{
+ return _filters.difference(_used_filters);
+}
diff --git a/engine/filters.hpp b/engine/filters.hpp
new file mode 100644
index 000000000000..91a667c3b46b
--- /dev/null
+++ b/engine/filters.hpp
@@ -0,0 +1,134 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/filters.hpp
+/// Representation and manipulation of filters for test cases.
+///
+/// All the filter classes in this module are supposed to be purely functional:
+/// they are mere filters that decide whether they match or not the input data
+/// fed to them. User-interface filter manipulation must go somewhere else.
+
+#if !defined(ENGINE_FILTERS_HPP)
+#define ENGINE_FILTERS_HPP
+
+#include "engine/filters_fwd.hpp"
+
+#include <ostream>
+#include <string>
+#include <set>
+#include <utility>
+
+#include "utils/fs/path.hpp"
+#include "utils/optional.ipp"
+
+
+namespace engine {
+
+
+/// Filter for test cases.
+///
+/// A filter is one of: the name of a directory containing test cases, the name
+/// of a test program, or the name of a test program plus the name of a test
+/// case.
+class test_filter {
+public:
+ /// The name of the test program or subdirectory to match.
+ utils::fs::path test_program;
+
+ /// The name of the test case to match; if empty, represents any test case.
+ std::string test_case;
+
+ test_filter(const utils::fs::path&, const std::string&);
+ static test_filter parse(const std::string&);
+
+ std::string str(void) const;
+
+ bool contains(const test_filter&) const;
+ bool matches_test_program(const utils::fs::path&) const;
+ bool matches_test_case(const utils::fs::path&, const std::string&) const;
+
+ bool operator<(const test_filter&) const;
+ bool operator==(const test_filter&) const;
+ bool operator!=(const test_filter&) const;
+};
+
+
+std::ostream& operator<<(std::ostream&, const test_filter&);
+
+
+/// Collection of user-provided filters to select test cases.
+///
+/// An empty collection of filters is considered to match any test case.
+///
+/// In general, the filters maintained by this class should be disjoint. If
+/// they are not, some filters may never have a chance to do a match, which is
+/// most likely the fault of the user. To check for non-disjoint filters before
+/// constructing this object, use check_disjoint_filters.
+class test_filters {
+ /// The user-provided filters.
+ std::set< test_filter > _filters;
+
+public:
+ explicit test_filters(const std::set< test_filter >&);
+
+ /// Return type of match_test_case. Indicates whether the filters have
+ /// matched a particular test case and, if they have, which filter did the
+ /// match (if any).
+ typedef std::pair< bool, utils::optional< test_filter > > match;
+
+ bool match_test_program(const utils::fs::path&) const;
+ match match_test_case(const utils::fs::path&, const std::string&) const;
+
+ std::set< test_filter > difference(const std::set< test_filter >&) const;
+};
+
+
+void check_disjoint_filters(const std::set< test_filter >&);
+
+
+/// Tracks state of the filters that have matched tests during execution.
+class filters_state {
+ /// The user-provided filters.
+ test_filters _filters;
+
+ /// Collection of filters that have matched test cases so far.
+ std::set< test_filter > _used_filters;
+
+public:
+ explicit filters_state(const std::set< test_filter >&);
+
+ bool match_test_program(const utils::fs::path&) const;
+ bool match_test_case(const utils::fs::path&, const std::string&);
+
+ std::set< test_filter > unused(void) const;
+};
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_FILTERS_HPP)
diff --git a/engine/filters_fwd.hpp b/engine/filters_fwd.hpp
new file mode 100644
index 000000000000..ee5d0c692ff5
--- /dev/null
+++ b/engine/filters_fwd.hpp
@@ -0,0 +1,45 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/filters_fwd.hpp
+/// Forward declarations for engine/filters.hpp
+
+#if !defined(ENGINE_FILTERS_FWD_HPP)
+#define ENGINE_FILTERS_FWD_HPP
+
+namespace engine {
+
+
+class filters_state;
+class test_filter;
+class test_filters;
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_FILTERS_FWD_HPP)
diff --git a/engine/filters_test.cpp b/engine/filters_test.cpp
new file mode 100644
index 000000000000..081755b2553f
--- /dev/null
+++ b/engine/filters_test.cpp
@@ -0,0 +1,594 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/filters.hpp"
+
+#include <stdexcept>
+
+#include <atf-c++.hpp>
+
+namespace fs = utils::fs;
+
+
+namespace {
+
+
+/// Syntactic sugar to instantiate engine::test_filter objects.
+///
+/// \param test_program Test program.
+/// \param test_case Test case.
+///
+/// \return A \p test_filter object, based on \p test_program and \p test_case.
+inline engine::test_filter
+mkfilter(const char* test_program, const char* test_case)
+{
+ return engine::test_filter(fs::path(test_program), test_case);
+}
+
+
+} // anonymous namespace
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__public_fields);
+ATF_TEST_CASE_BODY(test_filter__public_fields)
+{
+ const engine::test_filter filter(fs::path("foo/bar"), "baz");
+ ATF_REQUIRE_EQ(fs::path("foo/bar"), filter.test_program);
+ ATF_REQUIRE_EQ("baz", filter.test_case);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__ok);
+ATF_TEST_CASE_BODY(test_filter__parse__ok)
+{
+ const engine::test_filter filter(engine::test_filter::parse("foo"));
+ ATF_REQUIRE_EQ(fs::path("foo"), filter.test_program);
+ ATF_REQUIRE(filter.test_case.empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__empty);
+ATF_TEST_CASE_BODY(test_filter__parse__empty)
+{
+ ATF_REQUIRE_THROW_RE(std::runtime_error, "empty",
+ engine::test_filter::parse(""));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__absolute);
+ATF_TEST_CASE_BODY(test_filter__parse__absolute)
+{
+ ATF_REQUIRE_THROW_RE(std::runtime_error, "'/foo/bar'.*relative",
+ engine::test_filter::parse("/foo//bar"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__bad_program_name);
+ATF_TEST_CASE_BODY(test_filter__parse__bad_program_name)
+{
+ ATF_REQUIRE_THROW_RE(std::runtime_error, "Program name.*':foo'",
+ engine::test_filter::parse(":foo"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__bad_test_case);
+ATF_TEST_CASE_BODY(test_filter__parse__bad_test_case)
+{
+ ATF_REQUIRE_THROW_RE(std::runtime_error, "Test case.*'bar/baz:'",
+ engine::test_filter::parse("bar/baz:"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__bad_path);
+ATF_TEST_CASE_BODY(test_filter__parse__bad_path)
+{
+ // TODO(jmmv): Not implemented. At the moment, the only reason for a path
+ // to be invalid is if it is empty... but we are checking this exact
+ // condition ourselves as part of the input validation. So we can't mock in
+ // an argument with an invalid non-empty path...
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__str);
+ATF_TEST_CASE_BODY(test_filter__str)
+{
+ const engine::test_filter filter(fs::path("foo/bar"), "baz");
+ ATF_REQUIRE_EQ("foo/bar:baz", filter.str());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__contains__same);
+ATF_TEST_CASE_BODY(test_filter__contains__same)
+{
+ {
+ const engine::test_filter f(fs::path("foo/bar"), "baz");
+ ATF_REQUIRE(f.contains(f));
+ }
+ {
+ const engine::test_filter f(fs::path("foo/bar"), "");
+ ATF_REQUIRE(f.contains(f));
+ }
+ {
+ const engine::test_filter f(fs::path("foo"), "");
+ ATF_REQUIRE(f.contains(f));
+ }
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__contains__different);
+ATF_TEST_CASE_BODY(test_filter__contains__different)
+{
+ {
+ const engine::test_filter f1(fs::path("foo"), "");
+ const engine::test_filter f2(fs::path("foo"), "bar");
+ ATF_REQUIRE( f1.contains(f2));
+ ATF_REQUIRE(!f2.contains(f1));
+ }
+ {
+ const engine::test_filter f1(fs::path("foo/bar"), "");
+ const engine::test_filter f2(fs::path("foo/bar"), "baz");
+ ATF_REQUIRE( f1.contains(f2));
+ ATF_REQUIRE(!f2.contains(f1));
+ }
+ {
+ const engine::test_filter f1(fs::path("foo/bar"), "");
+ const engine::test_filter f2(fs::path("foo/baz"), "");
+ ATF_REQUIRE(!f1.contains(f2));
+ ATF_REQUIRE(!f2.contains(f1));
+ }
+ {
+ const engine::test_filter f1(fs::path("foo"), "");
+ const engine::test_filter f2(fs::path("foo/bar"), "");
+ ATF_REQUIRE( f1.contains(f2));
+ ATF_REQUIRE(!f2.contains(f1));
+ }
+ {
+ const engine::test_filter f1(fs::path("foo"), "bar");
+ const engine::test_filter f2(fs::path("foo/bar"), "");
+ ATF_REQUIRE(!f1.contains(f2));
+ ATF_REQUIRE(!f2.contains(f1));
+ }
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__matches_test_program)
+ATF_TEST_CASE_BODY(test_filter__matches_test_program)
+{
+ {
+ const engine::test_filter f(fs::path("top"), "unused");
+ ATF_REQUIRE( f.matches_test_program(fs::path("top")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("top2")));
+ }
+
+ {
+ const engine::test_filter f(fs::path("dir1/dir2"), "");
+ ATF_REQUIRE( f.matches_test_program(fs::path("dir1/dir2/foo")));
+ ATF_REQUIRE( f.matches_test_program(fs::path("dir1/dir2/bar")));
+ ATF_REQUIRE( f.matches_test_program(fs::path("dir1/dir2/bar/baz")));
+ ATF_REQUIRE( f.matches_test_program(fs::path("dir1/dir2/bar/baz")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("dir1")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/bar/baz")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("dir2/bar/baz")));
+ }
+
+ {
+ const engine::test_filter f(fs::path("dir1/dir2"), "unused");
+ ATF_REQUIRE( f.matches_test_program(fs::path("dir1/dir2")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/dir2/foo")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/dir2/bar")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/dir2/bar/baz")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/dir2/bar/baz")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("dir1")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/bar/baz")));
+ ATF_REQUIRE(!f.matches_test_program(fs::path("dir2/bar/baz")));
+ }
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__matches_test_case)
+ATF_TEST_CASE_BODY(test_filter__matches_test_case)
+{
+ {
+ const engine::test_filter f(fs::path("top"), "foo");
+ ATF_REQUIRE( f.matches_test_case(fs::path("top"), "foo"));
+ ATF_REQUIRE(!f.matches_test_case(fs::path("top"), "bar"));
+ }
+
+ {
+ const engine::test_filter f(fs::path("top"), "");
+ ATF_REQUIRE( f.matches_test_case(fs::path("top"), "foo"));
+ ATF_REQUIRE( f.matches_test_case(fs::path("top"), "bar"));
+ ATF_REQUIRE(!f.matches_test_case(fs::path("top2"), "foo"));
+ }
+
+ {
+ const engine::test_filter f(fs::path("d1/d2/prog"), "t1");
+ ATF_REQUIRE( f.matches_test_case(fs::path("d1/d2/prog"), "t1"));
+ ATF_REQUIRE(!f.matches_test_case(fs::path("d1/d2/prog"), "t2"));
+ }
+
+ {
+ const engine::test_filter f(fs::path("d1/d2"), "");
+ ATF_REQUIRE( f.matches_test_case(fs::path("d1/d2/prog"), "t1"));
+ ATF_REQUIRE( f.matches_test_case(fs::path("d1/d2/prog"), "t2"));
+ ATF_REQUIRE( f.matches_test_case(fs::path("d1/d2/prog2"), "t2"));
+ ATF_REQUIRE(!f.matches_test_case(fs::path("d1/d3"), "foo"));
+ ATF_REQUIRE(!f.matches_test_case(fs::path("d2"), "foo"));
+ }
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__operator_lt)
+ATF_TEST_CASE_BODY(test_filter__operator_lt)
+{
+ {
+ const engine::test_filter f1(fs::path("d1/d2"), "");
+ ATF_REQUIRE(!(f1 < f1));
+ }
+ {
+ const engine::test_filter f1(fs::path("d1/d2"), "");
+ const engine::test_filter f2(fs::path("d1/d3"), "");
+ ATF_REQUIRE( (f1 < f2));
+ ATF_REQUIRE(!(f2 < f1));
+ }
+ {
+ const engine::test_filter f1(fs::path("d1/d2"), "");
+ const engine::test_filter f2(fs::path("d1/d2"), "foo");
+ ATF_REQUIRE( (f1 < f2));
+ ATF_REQUIRE(!(f2 < f1));
+ }
+ {
+ const engine::test_filter f1(fs::path("d1/d2"), "bar");
+ const engine::test_filter f2(fs::path("d1/d2"), "foo");
+ ATF_REQUIRE( (f1 < f2));
+ ATF_REQUIRE(!(f2 < f1));
+ }
+ {
+ const engine::test_filter f1(fs::path("d1/d2"), "bar");
+ const engine::test_filter f2(fs::path("d1/d3"), "");
+ ATF_REQUIRE( (f1 < f2));
+ ATF_REQUIRE(!(f2 < f1));
+ }
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__operator_eq)
+ATF_TEST_CASE_BODY(test_filter__operator_eq)
+{
+ const engine::test_filter f1(fs::path("d1/d2"), "");
+ const engine::test_filter f2(fs::path("d1/d2"), "bar");
+ ATF_REQUIRE( (f1 == f1));
+ ATF_REQUIRE(!(f1 == f2));
+ ATF_REQUIRE(!(f2 == f1));
+ ATF_REQUIRE( (f2 == f2));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__operator_ne)
+ATF_TEST_CASE_BODY(test_filter__operator_ne)
+{
+ const engine::test_filter f1(fs::path("d1/d2"), "");
+ const engine::test_filter f2(fs::path("d1/d2"), "bar");
+ ATF_REQUIRE(!(f1 != f1));
+ ATF_REQUIRE( (f1 != f2));
+ ATF_REQUIRE( (f2 != f1));
+ ATF_REQUIRE(!(f2 != f2));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filter__output);
+ATF_TEST_CASE_BODY(test_filter__output)
+{
+ {
+ std::ostringstream str;
+ str << engine::test_filter(fs::path("d1/d2"), "");
+ ATF_REQUIRE_EQ(
+ "test_filter{test_program=d1/d2}",
+ str.str());
+ }
+ {
+ std::ostringstream str;
+ str << engine::test_filter(fs::path("d1/d2"), "bar");
+ ATF_REQUIRE_EQ(
+ "test_filter{test_program=d1/d2, test_case=bar}",
+ str.str());
+ }
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filters__match_test_case__no_filters)
+ATF_TEST_CASE_BODY(test_filters__match_test_case__no_filters)
+{
+ const std::set< engine::test_filter > raw_filters;
+
+ const engine::test_filters filters(raw_filters);
+ engine::test_filters::match match;
+
+ match = filters.match_test_case(fs::path("foo"), "baz");
+ ATF_REQUIRE(match.first);
+ ATF_REQUIRE(!match.second);
+
+ match = filters.match_test_case(fs::path("foo/bar"), "baz");
+ ATF_REQUIRE(match.first);
+ ATF_REQUIRE(!match.second);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filters__match_test_case__some_filters)
+ATF_TEST_CASE_BODY(test_filters__match_test_case__some_filters)
+{
+ std::set< engine::test_filter > raw_filters;
+ raw_filters.insert(mkfilter("top_test", ""));
+ raw_filters.insert(mkfilter("subdir_1", ""));
+ raw_filters.insert(mkfilter("subdir_2/a_test", ""));
+ raw_filters.insert(mkfilter("subdir_2/b_test", "foo"));
+
+ const engine::test_filters filters(raw_filters);
+ engine::test_filters::match match;
+
+ match = filters.match_test_case(fs::path("top_test"), "a");
+ ATF_REQUIRE(match.first);
+ ATF_REQUIRE_EQ("top_test", match.second.get().str());
+
+ match = filters.match_test_case(fs::path("subdir_1/foo"), "a");
+ ATF_REQUIRE(match.first);
+ ATF_REQUIRE_EQ("subdir_1", match.second.get().str());
+
+ match = filters.match_test_case(fs::path("subdir_1/bar"), "z");
+ ATF_REQUIRE(match.first);
+ ATF_REQUIRE_EQ("subdir_1", match.second.get().str());
+
+ match = filters.match_test_case(fs::path("subdir_2/a_test"), "bar");
+ ATF_REQUIRE(match.first);
+ ATF_REQUIRE_EQ("subdir_2/a_test", match.second.get().str());
+
+ match = filters.match_test_case(fs::path("subdir_2/b_test"), "foo");
+ ATF_REQUIRE(match.first);
+ ATF_REQUIRE_EQ("subdir_2/b_test:foo", match.second.get().str());
+
+ match = filters.match_test_case(fs::path("subdir_2/b_test"), "bar");
+ ATF_REQUIRE(!match.first);
+
+ match = filters.match_test_case(fs::path("subdir_2/c_test"), "foo");
+ ATF_REQUIRE(!match.first);
+
+ match = filters.match_test_case(fs::path("subdir_3"), "hello");
+ ATF_REQUIRE(!match.first);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filters__match_test_program__no_filters)
+ATF_TEST_CASE_BODY(test_filters__match_test_program__no_filters)
+{
+ const std::set< engine::test_filter > raw_filters;
+
+ const engine::test_filters filters(raw_filters);
+ ATF_REQUIRE(filters.match_test_program(fs::path("foo")));
+ ATF_REQUIRE(filters.match_test_program(fs::path("foo/bar")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filters__match_test_program__some_filters)
+ATF_TEST_CASE_BODY(test_filters__match_test_program__some_filters)
+{
+ std::set< engine::test_filter > raw_filters;
+ raw_filters.insert(mkfilter("top_test", ""));
+ raw_filters.insert(mkfilter("subdir_1", ""));
+ raw_filters.insert(mkfilter("subdir_2/a_test", ""));
+ raw_filters.insert(mkfilter("subdir_2/b_test", "foo"));
+
+ const engine::test_filters filters(raw_filters);
+ ATF_REQUIRE( filters.match_test_program(fs::path("top_test")));
+ ATF_REQUIRE( filters.match_test_program(fs::path("subdir_1/foo")));
+ ATF_REQUIRE( filters.match_test_program(fs::path("subdir_1/bar")));
+ ATF_REQUIRE( filters.match_test_program(fs::path("subdir_2/a_test")));
+ ATF_REQUIRE( filters.match_test_program(fs::path("subdir_2/b_test")));
+ ATF_REQUIRE(!filters.match_test_program(fs::path("subdir_2/c_test")));
+ ATF_REQUIRE(!filters.match_test_program(fs::path("subdir_3")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filters__difference__no_filters);
+ATF_TEST_CASE_BODY(test_filters__difference__no_filters)
+{
+ const std::set< engine::test_filter > in_filters;
+ const std::set< engine::test_filter > used;
+ const std::set< engine::test_filter > diff = engine::test_filters(
+ in_filters).difference(used);
+ ATF_REQUIRE(diff.empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filters__difference__some_filters__all_used);
+ATF_TEST_CASE_BODY(test_filters__difference__some_filters__all_used)
+{
+ std::set< engine::test_filter > in_filters;
+ in_filters.insert(mkfilter("a", ""));
+ in_filters.insert(mkfilter("b", "c"));
+
+ const std::set< engine::test_filter > used = in_filters;
+
+ const std::set< engine::test_filter > diff = engine::test_filters(
+ in_filters).difference(used);
+ ATF_REQUIRE(diff.empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test_filters__difference__some_filters__some_unused);
+ATF_TEST_CASE_BODY(test_filters__difference__some_filters__some_unused)
+{
+ std::set< engine::test_filter > in_filters;
+ in_filters.insert(mkfilter("a", ""));
+ in_filters.insert(mkfilter("b", "c"));
+ in_filters.insert(mkfilter("d", ""));
+ in_filters.insert(mkfilter("e", "f"));
+
+ std::set< engine::test_filter > used;
+ used.insert(mkfilter("b", "c"));
+ used.insert(mkfilter("d", ""));
+
+ const std::set< engine::test_filter > diff = engine::test_filters(
+ in_filters).difference(used);
+ ATF_REQUIRE_EQ(2, diff.size());
+ ATF_REQUIRE(diff.find(mkfilter("a", "")) != diff.end());
+ ATF_REQUIRE(diff.find(mkfilter("e", "f")) != diff.end());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_disjoint_filters__ok);
+ATF_TEST_CASE_BODY(check_disjoint_filters__ok)
+{
+ std::set< engine::test_filter > filters;
+ filters.insert(mkfilter("a", ""));
+ filters.insert(mkfilter("b", ""));
+ filters.insert(mkfilter("c", "a"));
+ filters.insert(mkfilter("c", "b"));
+
+ engine::check_disjoint_filters(filters);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_disjoint_filters__fail);
+ATF_TEST_CASE_BODY(check_disjoint_filters__fail)
+{
+ std::set< engine::test_filter > filters;
+ filters.insert(mkfilter("a", ""));
+ filters.insert(mkfilter("b", ""));
+ filters.insert(mkfilter("c", "a"));
+ filters.insert(mkfilter("d", "b"));
+ filters.insert(mkfilter("c", ""));
+
+ ATF_REQUIRE_THROW_RE(std::runtime_error, "'c'.*'c:a'.*not disjoint",
+ engine::check_disjoint_filters(filters));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(filters_state__match_test_program);
+ATF_TEST_CASE_BODY(filters_state__match_test_program)
+{
+ std::set< engine::test_filter > filters;
+ filters.insert(mkfilter("foo/bar", ""));
+ filters.insert(mkfilter("baz", "tc"));
+ engine::filters_state state(filters);
+
+ ATF_REQUIRE(state.match_test_program(fs::path("foo/bar/something")));
+ ATF_REQUIRE(state.match_test_program(fs::path("baz")));
+
+ ATF_REQUIRE(!state.match_test_program(fs::path("foo/baz")));
+ ATF_REQUIRE(!state.match_test_program(fs::path("hello")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(filters_state__match_test_case);
+ATF_TEST_CASE_BODY(filters_state__match_test_case)
+{
+ std::set< engine::test_filter > filters;
+ filters.insert(mkfilter("foo/bar", ""));
+ filters.insert(mkfilter("baz", "tc"));
+ engine::filters_state state(filters);
+
+ ATF_REQUIRE(state.match_test_case(fs::path("foo/bar/something"), "any"));
+ ATF_REQUIRE(state.match_test_case(fs::path("baz"), "tc"));
+
+ ATF_REQUIRE(!state.match_test_case(fs::path("foo/baz/something"), "tc"));
+ ATF_REQUIRE(!state.match_test_case(fs::path("baz"), "tc2"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(filters_state__unused__none);
+ATF_TEST_CASE_BODY(filters_state__unused__none)
+{
+ std::set< engine::test_filter > filters;
+ filters.insert(mkfilter("a/b", ""));
+ filters.insert(mkfilter("baz", "tc"));
+ filters.insert(mkfilter("hey/d", "yes"));
+ engine::filters_state state(filters);
+
+ state.match_test_case(fs::path("a/b/c"), "any");
+ state.match_test_case(fs::path("baz"), "tc");
+ state.match_test_case(fs::path("hey/d"), "yes");
+
+ ATF_REQUIRE(state.unused().empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(filters_state__unused__some);
+ATF_TEST_CASE_BODY(filters_state__unused__some)
+{
+ std::set< engine::test_filter > filters;
+ filters.insert(mkfilter("a/b", ""));
+ filters.insert(mkfilter("baz", "tc"));
+ filters.insert(mkfilter("hey/d", "yes"));
+ engine::filters_state state(filters);
+
+ state.match_test_program(fs::path("a/b/c"));
+ state.match_test_case(fs::path("baz"), "tc");
+
+ std::set< engine::test_filter > exp_unused;
+ exp_unused.insert(mkfilter("a/b", ""));
+ exp_unused.insert(mkfilter("hey/d", "yes"));
+
+ ATF_REQUIRE(exp_unused == state.unused());
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, test_filter__public_fields);
+ ATF_ADD_TEST_CASE(tcs, test_filter__parse__ok);
+ ATF_ADD_TEST_CASE(tcs, test_filter__parse__empty);
+ ATF_ADD_TEST_CASE(tcs, test_filter__parse__absolute);
+ ATF_ADD_TEST_CASE(tcs, test_filter__parse__bad_program_name);
+ ATF_ADD_TEST_CASE(tcs, test_filter__parse__bad_test_case);
+ ATF_ADD_TEST_CASE(tcs, test_filter__parse__bad_path);
+ ATF_ADD_TEST_CASE(tcs, test_filter__str);
+ ATF_ADD_TEST_CASE(tcs, test_filter__contains__same);
+ ATF_ADD_TEST_CASE(tcs, test_filter__contains__different);
+ ATF_ADD_TEST_CASE(tcs, test_filter__matches_test_program);
+ ATF_ADD_TEST_CASE(tcs, test_filter__matches_test_case);
+ ATF_ADD_TEST_CASE(tcs, test_filter__operator_lt);
+ ATF_ADD_TEST_CASE(tcs, test_filter__operator_eq);
+ ATF_ADD_TEST_CASE(tcs, test_filter__operator_ne);
+ ATF_ADD_TEST_CASE(tcs, test_filter__output);
+
+ ATF_ADD_TEST_CASE(tcs, test_filters__match_test_case__no_filters);
+ ATF_ADD_TEST_CASE(tcs, test_filters__match_test_case__some_filters);
+ ATF_ADD_TEST_CASE(tcs, test_filters__match_test_program__no_filters);
+ ATF_ADD_TEST_CASE(tcs, test_filters__match_test_program__some_filters);
+ ATF_ADD_TEST_CASE(tcs, test_filters__difference__no_filters);
+ ATF_ADD_TEST_CASE(tcs, test_filters__difference__some_filters__all_used);
+ ATF_ADD_TEST_CASE(tcs, test_filters__difference__some_filters__some_unused);
+
+ ATF_ADD_TEST_CASE(tcs, check_disjoint_filters__ok);
+ ATF_ADD_TEST_CASE(tcs, check_disjoint_filters__fail);
+
+ ATF_ADD_TEST_CASE(tcs, filters_state__match_test_program);
+ ATF_ADD_TEST_CASE(tcs, filters_state__match_test_case);
+ ATF_ADD_TEST_CASE(tcs, filters_state__unused__none);
+ ATF_ADD_TEST_CASE(tcs, filters_state__unused__some);
+}
diff --git a/engine/kyuafile.cpp b/engine/kyuafile.cpp
new file mode 100644
index 000000000000..4dca3193832b
--- /dev/null
+++ b/engine/kyuafile.cpp
@@ -0,0 +1,694 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/kyuafile.hpp"
+
+#include <algorithm>
+#include <iterator>
+#include <stdexcept>
+
+#include <lutok/exceptions.hpp>
+#include <lutok/operations.hpp>
+#include <lutok/stack_cleaner.hpp>
+#include <lutok/state.ipp>
+
+#include "engine/exceptions.hpp"
+#include "engine/scheduler.hpp"
+#include "model/metadata.hpp"
+#include "model/test_program.hpp"
+#include "utils/config/exceptions.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/datetime.hpp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/lua_module.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/logging/macros.hpp"
+#include "utils/noncopyable.hpp"
+#include "utils/optional.ipp"
+#include "utils/sanity.hpp"
+
+namespace config = utils::config;
+namespace datetime = utils::datetime;
+namespace fs = utils::fs;
+namespace scheduler = engine::scheduler;
+
+using utils::none;
+using utils::optional;
+
+
+// History of Kyuafile file versions:
+//
+// 3 - DOES NOT YET EXIST. Pending changes for when this is introduced:
+//
+// * Revisit what to do about the test_suite definition. Support for
+// per-test program overrides is deprecated and should be removed.
+// But, maybe, the whole test_suite definition idea is wrong and we
+// should instead be explicitly telling which configuration variables
+// to "inject" into each test program.
+//
+// 2 - Changed the syntax() call to take only a version number, instead of the
+// word 'config' as the first argument and the version as the second one.
+// Files now start with syntax(2) instead of syntax('kyuafile', 1).
+//
+// 1 - Initial version.
+
+
+namespace {
+
+
+static int lua_current_kyuafile(lutok::state&);
+static int lua_generic_test_program(lutok::state&);
+static int lua_include(lutok::state&);
+static int lua_syntax(lutok::state&);
+static int lua_test_suite(lutok::state&);
+
+
+/// Concatenates two paths while avoiding paths to start with './'.
+///
+/// \param root Path to the directory containing the file.
+/// \param file Path to concatenate to root. Cannot be absolute.
+///
+/// \return The concatenated path.
+static fs::path
+relativize(const fs::path& root, const fs::path& file)
+{
+ PRE(!file.is_absolute());
+
+ if (root == fs::path("."))
+ return file;
+ else
+ return root / file;
+}
+
+
+/// Implementation of a parser for Kyuafiles.
+///
+/// The main purpose of having this as a class is to keep track of global state
+/// within the Lua files and allowing the Lua callbacks to easily access such
+/// data.
+class parser : utils::noncopyable {
+ /// Lua state to parse a single Kyuafile file.
+ lutok::state _state;
+
+ /// Root directory of the test suite represented by the Kyuafile.
+ const fs::path _source_root;
+
+ /// Root directory of the test programs.
+ const fs::path _build_root;
+
+ /// Name of the Kyuafile to load relative to _source_root.
+ const fs::path _relative_filename;
+
+ /// Version of the Kyuafile file format requested by the parsed file.
+ ///
+ /// This is set once the Kyuafile invokes the syntax() call.
+ optional< int > _version;
+
+ /// Name of the test suite defined by the Kyuafile.
+ ///
+ /// This is set once the Kyuafile invokes the test_suite() call.
+ optional< std::string > _test_suite;
+
+ /// Collection of test programs defined by the Kyuafile.
+ ///
+ /// This acts as an accumulator for all the *_test_program() calls within
+ /// the Kyuafile.
+ model::test_programs_vector _test_programs;
+
+ /// Safely gets _test_suite and respects any test program overrides.
+ ///
+ /// \param program_override The test program-specific test suite name. May
+ /// be empty to indicate no override.
+ ///
+ /// \return The name of the test suite.
+ ///
+ /// \throw std::runtime_error If program_override is empty and the Kyuafile
+ /// did not yet define the global name of the test suite.
+ std::string
+ get_test_suite(const std::string& program_override)
+ {
+ std::string test_suite;
+
+ if (program_override.empty()) {
+ if (!_test_suite) {
+ throw std::runtime_error("No test suite defined in the "
+ "Kyuafile and no override provided in "
+ "the test_program definition");
+ }
+ test_suite = _test_suite.get();
+ } else {
+ test_suite = program_override;
+ }
+
+ return test_suite;
+ }
+
+public:
+ /// Initializes the parser and the Lua state.
+ ///
+ /// \param source_root_ The root directory of the test suite represented by
+ /// the Kyuafile.
+ /// \param build_root_ The root directory of the test programs.
+ /// \param relative_filename_ Name of the Kyuafile to load relative to
+ /// source_root_.
+ /// \param user_config User configuration holding any test suite properties
+ /// to be passed to the list operation.
+ /// \param scheduler_handle The scheduler context to use for loading the
+ /// test case lists.
+ parser(const fs::path& source_root_, const fs::path& build_root_,
+ const fs::path& relative_filename_,
+ const config::tree& user_config,
+ scheduler::scheduler_handle& scheduler_handle) :
+ _source_root(source_root_), _build_root(build_root_),
+ _relative_filename(relative_filename_)
+ {
+ lutok::stack_cleaner cleaner(_state);
+
+ _state.push_cxx_function(lua_syntax);
+ _state.set_global("syntax");
+
+ *_state.new_userdata< parser* >() = this;
+ _state.set_global("_parser");
+
+ _state.push_cxx_function(lua_current_kyuafile);
+ _state.set_global("current_kyuafile");
+
+ *_state.new_userdata< const config::tree* >() = &user_config;
+ *_state.new_userdata< scheduler::scheduler_handle* >() =
+ &scheduler_handle;
+ _state.push_cxx_closure(lua_include, 2);
+ _state.set_global("include");
+
+ _state.push_cxx_function(lua_test_suite);
+ _state.set_global("test_suite");
+
+ const std::set< std::string > interfaces =
+ scheduler::registered_interface_names();
+ for (std::set< std::string >::const_iterator iter = interfaces.begin();
+ iter != interfaces.end(); ++iter) {
+ const std::string& interface = *iter;
+
+ _state.push_string(interface);
+ *_state.new_userdata< const config::tree* >() = &user_config;
+ *_state.new_userdata< scheduler::scheduler_handle* >() =
+ &scheduler_handle;
+ _state.push_cxx_closure(lua_generic_test_program, 3);
+ _state.set_global(interface + "_test_program");
+ }
+
+ _state.open_base();
+ _state.open_string();
+ _state.open_table();
+ fs::open_fs(_state, callback_current_kyuafile().branch_path());
+ }
+
+ /// Destructor.
+ ~parser(void)
+ {
+ }
+
+ /// Gets the parser object associated to a Lua state.
+ ///
+ /// \param state The Lua state from which to obtain the parser object.
+ ///
+ /// \return A pointer to the parser.
+ static parser*
+ get_from_state(lutok::state& state)
+ {
+ lutok::stack_cleaner cleaner(state);
+ state.get_global("_parser");
+ return *state.to_userdata< parser* >(-1);
+ }
+
+ /// Callback for the Kyuafile current_kyuafile() function.
+ ///
+ /// \return Returns the absolute path to the current Kyuafile.
+ fs::path
+ callback_current_kyuafile(void) const
+ {
+ const fs::path file = relativize(_source_root, _relative_filename);
+ if (file.is_absolute())
+ return file;
+ else
+ return file.to_absolute();
+ }
+
+ /// Callback for the Kyuafile include() function.
+ ///
+ /// \post _test_programs is extended with the the test programs defined by
+ /// the included file.
+ ///
+ /// \param raw_file Path to the file to include.
+ /// \param user_config User configuration holding any test suite properties
+ /// to be passed to the list operation.
+ /// \param scheduler_handle Scheduler context to run test programs in.
+ void
+ callback_include(const fs::path& raw_file,
+ const config::tree& user_config,
+ scheduler::scheduler_handle& scheduler_handle)
+ {
+ const fs::path file = relativize(_relative_filename.branch_path(),
+ raw_file);
+ const model::test_programs_vector subtps =
+ parser(_source_root, _build_root, file, user_config,
+ scheduler_handle).parse();
+
+ std::copy(subtps.begin(), subtps.end(),
+ std::back_inserter(_test_programs));
+ }
+
+ /// Callback for the Kyuafile syntax() function.
+ ///
+ /// \post _version is set to the requested version.
+ ///
+ /// \param version Version of the Kyuafile syntax requested by the file.
+ ///
+ /// \throw std::runtime_error If the format or the version are invalid, or
+ /// if syntax() has already been called.
+ void
+ callback_syntax(const int version)
+ {
+ if (_version)
+ throw std::runtime_error("Can only call syntax() once");
+
+ if (version < 1 || version > 2)
+ throw std::runtime_error(F("Unsupported file version %s") %
+ version);
+
+ _version = utils::make_optional(version);
+ }
+
+ /// Callback for the various Kyuafile *_test_program() functions.
+ ///
+ /// \post _test_programs is extended to include the newly defined test
+ /// program.
+ ///
+ /// \param interface Name of the test program interface.
+ /// \param raw_path Path to the test program, relative to the Kyuafile.
+ /// This has to be adjusted according to the relative location of this
+ /// Kyuafile to _source_root.
+ /// \param test_suite_override Name of the test suite this test program
+ /// belongs to, if explicitly defined at the test program level.
+ /// \param metadata Metadata variables passed to the test program.
+ /// \param user_config User configuration holding any test suite properties
+ /// to be passed to the list operation.
+ /// \param scheduler_handle Scheduler context to run test programs in.
+ ///
+ /// \throw std::runtime_error If the test program definition is invalid or
+ /// if the test program does not exist.
+ void
+ callback_test_program(const std::string& interface,
+ const fs::path& raw_path,
+ const std::string& test_suite_override,
+ const model::metadata& metadata,
+ const config::tree& user_config,
+ scheduler::scheduler_handle& scheduler_handle)
+ {
+ if (raw_path.is_absolute())
+ throw std::runtime_error(F("Got unexpected absolute path for test "
+ "program '%s'") % raw_path);
+ else if (raw_path.str() != raw_path.leaf_name())
+ throw std::runtime_error(F("Test program '%s' cannot contain path "
+ "components") % raw_path);
+
+ const fs::path path = relativize(_relative_filename.branch_path(),
+ raw_path);
+
+ if (!fs::exists(_build_root / path))
+ throw std::runtime_error(F("Non-existent test program '%s'") %
+ path);
+
+ const std::string test_suite = get_test_suite(test_suite_override);
+
+ _test_programs.push_back(model::test_program_ptr(
+ new scheduler::lazy_test_program(interface, path, _build_root,
+ test_suite, metadata, user_config,
+ scheduler_handle)));
+ }
+
+ /// Callback for the Kyuafile test_suite() function.
+ ///
+ /// \post _version is set to the requested version.
+ ///
+ /// \param name Name of the test suite.
+ ///
+ /// \throw std::runtime_error If test_suite() has already been called.
+ void
+ callback_test_suite(const std::string& name)
+ {
+ if (_test_suite)
+ throw std::runtime_error("Can only call test_suite() once");
+ _test_suite = utils::make_optional(name);
+ }
+
+ /// Parses the Kyuafile.
+ ///
+ /// \pre Can only be invoked once.
+ ///
+ /// \return The collection of test programs defined by the Kyuafile.
+ ///
+ /// \throw load_error If there is any problem parsing the file.
+ const model::test_programs_vector&
+ parse(void)
+ {
+ PRE(_test_programs.empty());
+
+ const fs::path load_path = relativize(_source_root, _relative_filename);
+ try {
+ lutok::do_file(_state, load_path.str(), 0, 0, 0);
+ } catch (const std::runtime_error& e) {
+ // It is tempting to think that all of our various auxiliary
+ // functions above could raise load_error by themselves thus making
+ // this exception rewriting here unnecessary. Howver, that would
+ // not work because the helper functions above are executed within a
+ // Lua context, and we lose their type when they are propagated out
+ // of it.
+ throw engine::load_error(load_path, e.what());
+ }
+
+ if (!_version)
+ throw engine::load_error(load_path, "syntax() never called");
+
+ return _test_programs;
+ }
+};
+
+
+/// Glue to invoke parser::callback_test_program() from Lua.
+///
+/// This is a helper function for the various *_test_program() calls, as they
+/// only differ in the interface of the defined test program.
+///
+/// \pre state(-1) A table with the arguments that define the test program. The
+/// special argument 'test_suite' provides an override to the global test suite
+/// name. The rest of the arguments are part of the test program metadata.
+/// \pre state(upvalue 1) String with the name of the interface.
+/// \pre state(upvalue 2) User configuration with the per-test suite settings.
+/// \pre state(upvalue 3) Scheduler context to run test programs in.
+///
+/// \param state The Lua state that executed the function.
+///
+/// \return Number of return values left on the Lua stack.
+///
+/// \throw std::runtime_error If the arguments to the function are invalid.
+static int
+lua_generic_test_program(lutok::state& state)
+{
+ if (!state.is_string(state.upvalue_index(1)))
+ throw std::runtime_error("Found corrupt state for test_program "
+ "function");
+ const std::string interface = state.to_string(state.upvalue_index(1));
+
+ if (!state.is_userdata(state.upvalue_index(2)))
+ throw std::runtime_error("Found corrupt state for test_program "
+ "function");
+ const config::tree* user_config = *state.to_userdata< const config::tree* >(
+ state.upvalue_index(2));
+
+ if (!state.is_userdata(state.upvalue_index(3)))
+ throw std::runtime_error("Found corrupt state for test_program "
+ "function");
+ scheduler::scheduler_handle* scheduler_handle =
+ *state.to_userdata< scheduler::scheduler_handle* >(
+ state.upvalue_index(3));
+
+ if (!state.is_table(-1))
+ throw std::runtime_error(
+ F("%s_test_program expects a table of properties as its single "
+ "argument") % interface);
+
+ scheduler::ensure_valid_interface(interface);
+
+ lutok::stack_cleaner cleaner(state);
+
+ state.push_string("name");
+ state.get_table(-2);
+ if (!state.is_string(-1))
+ throw std::runtime_error("Test program name not defined or not a "
+ "string");
+ const fs::path path(state.to_string(-1));
+ state.pop(1);
+
+ state.push_string("test_suite");
+ state.get_table(-2);
+ std::string test_suite;
+ if (state.is_nil(-1)) {
+ // Leave empty to use the global test-suite value.
+ } else if (state.is_string(-1)) {
+ test_suite = state.to_string(-1);
+ } else {
+ throw std::runtime_error(F("Found non-string value in the test_suite "
+ "property of test program '%s'") % path);
+ }
+ state.pop(1);
+
+ model::metadata_builder mdbuilder;
+ state.push_nil();
+ while (state.next(-2)) {
+ if (!state.is_string(-2))
+ throw std::runtime_error(F("Found non-string metadata property "
+ "name in test program '%s'") %
+ path);
+ const std::string property = state.to_string(-2);
+
+ if (property != "name" && property != "test_suite") {
+ std::string value;
+ if (state.is_boolean(-1)) {
+ value = F("%s") % state.to_boolean(-1);
+ } else if (state.is_number(-1)) {
+ value = F("%s") % state.to_integer(-1);
+ } else if (state.is_string(-1)) {
+ value = state.to_string(-1);
+ } else {
+ throw std::runtime_error(
+ F("Metadata property '%s' in test program '%s' cannot be "
+ "converted to a string") % property % path);
+ }
+
+ mdbuilder.set_string(property, value);
+ }
+
+ state.pop(1);
+ }
+
+ parser::get_from_state(state)->callback_test_program(
+ interface, path, test_suite, mdbuilder.build(), *user_config,
+ *scheduler_handle);
+ return 0;
+}
+
+
+/// Glue to invoke parser::callback_current_kyuafile() from Lua.
+///
+/// \param state The Lua state that executed the function.
+///
+/// \return Number of return values left on the Lua stack.
+static int
+lua_current_kyuafile(lutok::state& state)
+{
+ state.push_string(parser::get_from_state(state)->
+ callback_current_kyuafile().str());
+ return 1;
+}
+
+
+/// Glue to invoke parser::callback_include() from Lua.
+///
+/// \param state The Lua state that executed the function.
+///
+/// \pre state(upvalue 1) User configuration with the per-test suite settings.
+/// \pre state(upvalue 2) Scheduler context to run test programs in.
+///
+/// \return Number of return values left on the Lua stack.
+static int
+lua_include(lutok::state& state)
+{
+ if (!state.is_userdata(state.upvalue_index(1)))
+ throw std::runtime_error("Found corrupt state for test_program "
+ "function");
+ const config::tree* user_config = *state.to_userdata< const config::tree* >(
+ state.upvalue_index(1));
+
+ if (!state.is_userdata(state.upvalue_index(2)))
+ throw std::runtime_error("Found corrupt state for test_program "
+ "function");
+ scheduler::scheduler_handle* scheduler_handle =
+ *state.to_userdata< scheduler::scheduler_handle* >(
+ state.upvalue_index(2));
+
+ parser::get_from_state(state)->callback_include(
+ fs::path(state.to_string(-1)), *user_config, *scheduler_handle);
+ return 0;
+}
+
+
+/// Glue to invoke parser::callback_syntax() from Lua.
+///
+/// \pre state(-2) The syntax format name, if a v1 file.
+/// \pre state(-1) The syntax format version.
+///
+/// \param state The Lua state that executed the function.
+///
+/// \return Number of return values left on the Lua stack.
+static int
+lua_syntax(lutok::state& state)
+{
+ if (!state.is_number(-1))
+ throw std::runtime_error("Last argument to syntax must be a number");
+ const int syntax_version = state.to_integer(-1);
+
+ if (syntax_version == 1) {
+ if (state.get_top() != 2)
+ throw std::runtime_error("Version 1 files need two arguments to "
+ "syntax()");
+ if (!state.is_string(-2) || state.to_string(-2) != "kyuafile")
+ throw std::runtime_error("First argument to syntax must be "
+ "'kyuafile' for version 1 files");
+ } else {
+ if (state.get_top() != 1)
+ throw std::runtime_error("syntax() only takes one argument");
+ }
+
+ parser::get_from_state(state)->callback_syntax(syntax_version);
+ return 0;
+}
+
+
+/// Glue to invoke parser::callback_test_suite() from Lua.
+///
+/// \param state The Lua state that executed the function.
+///
+/// \return Number of return values left on the Lua stack.
+static int
+lua_test_suite(lutok::state& state)
+{
+ parser::get_from_state(state)->callback_test_suite(state.to_string(-1));
+ return 0;
+}
+
+
+} // anonymous namespace
+
+
+/// Constructs a kyuafile form initialized data.
+///
+/// Use load() to parse a test suite configuration file and construct a
+/// kyuafile object.
+///
+/// \param source_root_ The root directory for the test suite represented by the
+/// Kyuafile. In other words, the directory containing the first Kyuafile
+/// processed.
+/// \param build_root_ The root directory for the test programs themselves. In
+/// general, this will be the same as source_root_. If different, the
+/// specified directory must follow the exact same layout of source_root_.
+/// \param tps_ Collection of test programs that belong to this test suite.
+engine::kyuafile::kyuafile(const fs::path& source_root_,
+ const fs::path& build_root_,
+ const model::test_programs_vector& tps_) :
+ _source_root(source_root_),
+ _build_root(build_root_),
+ _test_programs(tps_)
+{
+}
+
+
+/// Destructor.
+engine::kyuafile::~kyuafile(void)
+{
+}
+
+
+/// Parses a test suite configuration file.
+///
+/// \param file The file to parse.
+/// \param user_build_root If not none, specifies a path to a directory
+/// containing the test programs themselves. The layout of the build root
+/// must match the layout of the source root (which is just the directory
+/// from which the Kyuafile is being read).
+/// \param user_config User configuration holding any test suite properties
+/// to be passed to the list operation.
+/// \param scheduler_handle The scheduler context to use for loading the test
+/// case lists.
+///
+/// \return High-level representation of the configuration file.
+///
+/// \throw load_error If there is any problem loading the file. This includes
+/// file access errors and syntax errors.
+engine::kyuafile
+engine::kyuafile::load(const fs::path& file,
+ const optional< fs::path > user_build_root,
+ const config::tree& user_config,
+ scheduler::scheduler_handle& scheduler_handle)
+{
+ const fs::path source_root_ = file.branch_path();
+ const fs::path build_root_ = user_build_root ?
+ user_build_root.get() : source_root_;
+
+ // test_program.absolute_path() uses the current work directory and that
+ // fails to resolve the correct path once we have used chdir to enter the
+ // test work directory. To prevent this causing issues down the road,
+ // force the build root to be absolute so that absolute_path() does not
+ // need to rely on the current work directory.
+ const fs::path abs_build_root = build_root_.is_absolute() ?
+ build_root_ : build_root_.to_absolute();
+
+ return kyuafile(source_root_, build_root_,
+ parser(source_root_, abs_build_root,
+ fs::path(file.leaf_name()), user_config,
+ scheduler_handle).parse());
+}
+
+
+/// Gets the root directory of the test suite.
+///
+/// \return A path.
+const fs::path&
+engine::kyuafile::source_root(void) const
+{
+ return _source_root;
+}
+
+
+/// Gets the root directory of the test programs.
+///
+/// \return A path.
+const fs::path&
+engine::kyuafile::build_root(void) const
+{
+ return _build_root;
+}
+
+
+/// Gets the collection of test programs that belong to this test suite.
+///
+/// \return Collection of test program executable names.
+const model::test_programs_vector&
+engine::kyuafile::test_programs(void) const
+{
+ return _test_programs;
+}
diff --git a/engine/kyuafile.hpp b/engine/kyuafile.hpp
new file mode 100644
index 000000000000..161f4305f4d1
--- /dev/null
+++ b/engine/kyuafile.hpp
@@ -0,0 +1,96 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/kyuafile.hpp
+/// Test suite configuration parsing and representation.
+
+#if !defined(ENGINE_KYUAFILE_HPP)
+#define ENGINE_KYUAFILE_HPP
+
+#include "engine/kyuafile_fwd.hpp"
+
+#include <string>
+#include <vector>
+
+#include <lutok/state.hpp>
+
+#include "engine/scheduler_fwd.hpp"
+#include "model/test_program_fwd.hpp"
+#include "utils/config/tree_fwd.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/optional_fwd.hpp"
+
+namespace engine {
+
+
+/// Representation of the configuration of a test suite.
+///
+/// Test suites are collections of related test programs. They are described by
+/// a configuration file.
+///
+/// Test suites have two path references: one to the "source root" and another
+/// one to the "build root". The source root points to the directory from which
+/// the Kyuafile is being read, and all recursive inclusions are resolved
+/// relative to that directory. The build root points to the directory
+/// containing the generated test programs and is prepended to the absolute path
+/// of the test programs referenced by the Kyuafiles. In general, the build
+/// root will be the same as the source root; however, when using a build system
+/// that supports "build directories", providing this option comes in handy to
+/// allow running the tests without much hassle.
+///
+/// This class provides the parser for test suite configuration files and
+/// methods to access the parsed data.
+class kyuafile {
+ /// Path to the directory containing the top-level Kyuafile loaded.
+ utils::fs::path _source_root;
+
+ /// Path to the directory containing the test programs.
+ utils::fs::path _build_root;
+
+ /// Collection of the test programs defined in the Kyuafile.
+ model::test_programs_vector _test_programs;
+
+public:
+ explicit kyuafile(const utils::fs::path&, const utils::fs::path&,
+ const model::test_programs_vector&);
+ ~kyuafile(void);
+
+ static kyuafile load(const utils::fs::path&,
+ const utils::optional< utils::fs::path >,
+ const utils::config::tree&,
+ scheduler::scheduler_handle&);
+
+ const utils::fs::path& source_root(void) const;
+ const utils::fs::path& build_root(void) const;
+ const model::test_programs_vector& test_programs(void) const;
+};
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_KYUAFILE_HPP)
diff --git a/engine/kyuafile_fwd.hpp b/engine/kyuafile_fwd.hpp
new file mode 100644
index 000000000000..60a98f65e3ab
--- /dev/null
+++ b/engine/kyuafile_fwd.hpp
@@ -0,0 +1,43 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/kyuafile_fwd.hpp
+/// Forward declarations for engine/kyuafile.hpp
+
+#if !defined(ENGINE_KYUAFILE_FWD_HPP)
+#define ENGINE_KYUAFILE_FWD_HPP
+
+namespace engine {
+
+
+class kyuafile;
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_KYUAFILE_FWD_HPP)
diff --git a/engine/kyuafile_test.cpp b/engine/kyuafile_test.cpp
new file mode 100644
index 000000000000..d95f28c71acb
--- /dev/null
+++ b/engine/kyuafile_test.cpp
@@ -0,0 +1,606 @@
+// Copyright 2010 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/kyuafile.hpp"
+
+extern "C" {
+#include <unistd.h>
+}
+
+#include <stdexcept>
+#include <typeinfo>
+
+#include <atf-c++.hpp>
+#include <lutok/operations.hpp>
+#include <lutok/state.ipp>
+#include <lutok/test_utils.hpp>
+
+#include "engine/atf.hpp"
+#include "engine/exceptions.hpp"
+#include "engine/plain.hpp"
+#include "engine/scheduler.hpp"
+#include "engine/tap.hpp"
+#include "model/metadata.hpp"
+#include "model/test_program.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/datetime.hpp"
+#include "utils/env.hpp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/optional.ipp"
+
+namespace config = utils::config;
+namespace datetime = utils::datetime;
+namespace fs = utils::fs;
+namespace scheduler = engine::scheduler;
+
+using utils::none;
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__empty);
+ATF_TEST_CASE_BODY(kyuafile__load__empty)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ atf::utils::create_file("config", "syntax(2)\n");
+
+ const engine::kyuafile suite = engine::kyuafile::load(
+ fs::path("config"), none, config::tree(), handle);
+ ATF_REQUIRE_EQ(fs::path("."), suite.source_root());
+ ATF_REQUIRE_EQ(fs::path("."), suite.build_root());
+ ATF_REQUIRE_EQ(0, suite.test_programs().size());
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__real_interfaces);
+ATF_TEST_CASE_BODY(kyuafile__load__real_interfaces)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "test_suite('one-suite')\n"
+ "atf_test_program{name='1st'}\n"
+ "atf_test_program{name='2nd', test_suite='first'}\n"
+ "plain_test_program{name='3rd'}\n"
+ "tap_test_program{name='4th', test_suite='second'}\n"
+ "include('dir/config')\n");
+
+ fs::mkdir(fs::path("dir"), 0755);
+ atf::utils::create_file(
+ "dir/config",
+ "syntax(2)\n"
+ "atf_test_program{name='1st', test_suite='other-suite'}\n"
+ "include('subdir/config')\n");
+
+ fs::mkdir(fs::path("dir/subdir"), 0755);
+ atf::utils::create_file(
+ "dir/subdir/config",
+ "syntax(2)\n"
+ "atf_test_program{name='5th', test_suite='last-suite'}\n");
+
+ atf::utils::create_file("1st", "");
+ atf::utils::create_file("2nd", "");
+ atf::utils::create_file("3rd", "");
+ atf::utils::create_file("4th", "");
+ atf::utils::create_file("dir/1st", "");
+ atf::utils::create_file("dir/subdir/5th", "");
+
+ const engine::kyuafile suite = engine::kyuafile::load(
+ fs::path("config"), none, config::tree(), handle);
+ ATF_REQUIRE_EQ(fs::path("."), suite.source_root());
+ ATF_REQUIRE_EQ(fs::path("."), suite.build_root());
+ ATF_REQUIRE_EQ(6, suite.test_programs().size());
+
+ ATF_REQUIRE_EQ("atf", suite.test_programs()[0]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("1st"), suite.test_programs()[0]->relative_path());
+ ATF_REQUIRE_EQ("one-suite", suite.test_programs()[0]->test_suite_name());
+
+ ATF_REQUIRE_EQ("atf", suite.test_programs()[1]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("2nd"), suite.test_programs()[1]->relative_path());
+ ATF_REQUIRE_EQ("first", suite.test_programs()[1]->test_suite_name());
+
+ ATF_REQUIRE_EQ("plain", suite.test_programs()[2]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("3rd"), suite.test_programs()[2]->relative_path());
+ ATF_REQUIRE_EQ("one-suite", suite.test_programs()[2]->test_suite_name());
+
+ ATF_REQUIRE_EQ("tap", suite.test_programs()[3]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("4th"), suite.test_programs()[3]->relative_path());
+ ATF_REQUIRE_EQ("second", suite.test_programs()[3]->test_suite_name());
+
+ ATF_REQUIRE_EQ("atf", suite.test_programs()[4]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("dir/1st"),
+ suite.test_programs()[4]->relative_path());
+ ATF_REQUIRE_EQ("other-suite", suite.test_programs()[4]->test_suite_name());
+
+ ATF_REQUIRE_EQ("atf", suite.test_programs()[5]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("dir/subdir/5th"),
+ suite.test_programs()[5]->relative_path());
+ ATF_REQUIRE_EQ("last-suite", suite.test_programs()[5]->test_suite_name());
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__mock_interfaces);
+ATF_TEST_CASE_BODY(kyuafile__load__mock_interfaces)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ std::shared_ptr< scheduler::interface > mock_interface(
+ new engine::plain_interface());
+
+ scheduler::register_interface("some", mock_interface);
+ scheduler::register_interface("random", mock_interface);
+ scheduler::register_interface("names", mock_interface);
+
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "test_suite('one-suite')\n"
+ "some_test_program{name='1st'}\n"
+ "random_test_program{name='2nd'}\n"
+ "names_test_program{name='3rd'}\n");
+
+ atf::utils::create_file("1st", "");
+ atf::utils::create_file("2nd", "");
+ atf::utils::create_file("3rd", "");
+
+ const engine::kyuafile suite = engine::kyuafile::load(
+ fs::path("config"), none, config::tree(), handle);
+ ATF_REQUIRE_EQ(fs::path("."), suite.source_root());
+ ATF_REQUIRE_EQ(fs::path("."), suite.build_root());
+ ATF_REQUIRE_EQ(3, suite.test_programs().size());
+
+ ATF_REQUIRE_EQ("some", suite.test_programs()[0]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("1st"), suite.test_programs()[0]->relative_path());
+
+ ATF_REQUIRE_EQ("random", suite.test_programs()[1]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("2nd"), suite.test_programs()[1]->relative_path());
+
+ ATF_REQUIRE_EQ("names", suite.test_programs()[2]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("3rd"), suite.test_programs()[2]->relative_path());
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__metadata);
+ATF_TEST_CASE_BODY(kyuafile__load__metadata)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "atf_test_program{name='1st', test_suite='first',"
+ " allowed_architectures='amd64 i386', timeout=15}\n"
+ "plain_test_program{name='2nd', test_suite='second',"
+ " required_files='foo /bar//baz', required_user='root',"
+ " ['custom.a-number']=123, ['custom.a-bool']=true}\n");
+ atf::utils::create_file("1st", "");
+ atf::utils::create_file("2nd", "");
+
+ const engine::kyuafile suite = engine::kyuafile::load(
+ fs::path("config"), none, config::tree(), handle);
+ ATF_REQUIRE_EQ(2, suite.test_programs().size());
+
+ ATF_REQUIRE_EQ("atf", suite.test_programs()[0]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("1st"), suite.test_programs()[0]->relative_path());
+ ATF_REQUIRE_EQ("first", suite.test_programs()[0]->test_suite_name());
+ const model::metadata md1 = model::metadata_builder()
+ .add_allowed_architecture("amd64")
+ .add_allowed_architecture("i386")
+ .set_timeout(datetime::delta(15, 0))
+ .build();
+ ATF_REQUIRE_EQ(md1, suite.test_programs()[0]->get_metadata());
+
+ ATF_REQUIRE_EQ("plain", suite.test_programs()[1]->interface_name());
+ ATF_REQUIRE_EQ(fs::path("2nd"), suite.test_programs()[1]->relative_path());
+ ATF_REQUIRE_EQ("second", suite.test_programs()[1]->test_suite_name());
+ const model::metadata md2 = model::metadata_builder()
+ .add_required_file(fs::path("foo"))
+ .add_required_file(fs::path("/bar/baz"))
+ .add_custom("a-bool", "true")
+ .add_custom("a-number", "123")
+ .set_required_user("root")
+ .build();
+ ATF_REQUIRE_EQ(md2, suite.test_programs()[1]->get_metadata());
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__current_directory);
+ATF_TEST_CASE_BODY(kyuafile__load__current_directory)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "atf_test_program{name='one', test_suite='first'}\n"
+ "include('config2')\n");
+
+ atf::utils::create_file(
+ "config2",
+ "syntax(2)\n"
+ "test_suite('second')\n"
+ "atf_test_program{name='two'}\n");
+
+ atf::utils::create_file("one", "");
+ atf::utils::create_file("two", "");
+
+ const engine::kyuafile suite = engine::kyuafile::load(
+ fs::path("config"), none, config::tree(), handle);
+ ATF_REQUIRE_EQ(fs::path("."), suite.source_root());
+ ATF_REQUIRE_EQ(fs::path("."), suite.build_root());
+ ATF_REQUIRE_EQ(2, suite.test_programs().size());
+ ATF_REQUIRE_EQ(fs::path("one"), suite.test_programs()[0]->relative_path());
+ ATF_REQUIRE_EQ("first", suite.test_programs()[0]->test_suite_name());
+ ATF_REQUIRE_EQ(fs::path("two"),
+ suite.test_programs()[1]->relative_path());
+ ATF_REQUIRE_EQ("second", suite.test_programs()[1]->test_suite_name());
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__other_directory);
+ATF_TEST_CASE_BODY(kyuafile__load__other_directory)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ fs::mkdir(fs::path("root"), 0755);
+ atf::utils::create_file(
+ "root/config",
+ "syntax(2)\n"
+ "test_suite('abc')\n"
+ "atf_test_program{name='one'}\n"
+ "include('dir/config')\n");
+
+ fs::mkdir(fs::path("root/dir"), 0755);
+ atf::utils::create_file(
+ "root/dir/config",
+ "syntax(2)\n"
+ "test_suite('foo')\n"
+ "atf_test_program{name='two', test_suite='def'}\n"
+ "atf_test_program{name='three'}\n");
+
+ atf::utils::create_file("root/one", "");
+ atf::utils::create_file("root/dir/two", "");
+ atf::utils::create_file("root/dir/three", "");
+
+ const engine::kyuafile suite = engine::kyuafile::load(
+ fs::path("root/config"), none, config::tree(), handle);
+ ATF_REQUIRE_EQ(fs::path("root"), suite.source_root());
+ ATF_REQUIRE_EQ(fs::path("root"), suite.build_root());
+ ATF_REQUIRE_EQ(3, suite.test_programs().size());
+ ATF_REQUIRE_EQ(fs::path("one"), suite.test_programs()[0]->relative_path());
+ ATF_REQUIRE_EQ("abc", suite.test_programs()[0]->test_suite_name());
+ ATF_REQUIRE_EQ(fs::path("dir/two"),
+ suite.test_programs()[1]->relative_path());
+ ATF_REQUIRE_EQ("def", suite.test_programs()[1]->test_suite_name());
+ ATF_REQUIRE_EQ(fs::path("dir/three"),
+ suite.test_programs()[2]->relative_path());
+ ATF_REQUIRE_EQ("foo", suite.test_programs()[2]->test_suite_name());
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__build_directory);
+ATF_TEST_CASE_BODY(kyuafile__load__build_directory)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ fs::mkdir(fs::path("srcdir"), 0755);
+ atf::utils::create_file(
+ "srcdir/config",
+ "syntax(2)\n"
+ "test_suite('abc')\n"
+ "atf_test_program{name='one'}\n"
+ "include('dir/config')\n");
+
+ fs::mkdir(fs::path("srcdir/dir"), 0755);
+ atf::utils::create_file(
+ "srcdir/dir/config",
+ "syntax(2)\n"
+ "test_suite('foo')\n"
+ "atf_test_program{name='two', test_suite='def'}\n"
+ "atf_test_program{name='three'}\n");
+
+ fs::mkdir(fs::path("builddir"), 0755);
+ atf::utils::create_file("builddir/one", "");
+ fs::mkdir(fs::path("builddir/dir"), 0755);
+ atf::utils::create_file("builddir/dir/two", "");
+ atf::utils::create_file("builddir/dir/three", "");
+
+ const engine::kyuafile suite = engine::kyuafile::load(
+ fs::path("srcdir/config"), utils::make_optional(fs::path("builddir")),
+ config::tree(), handle);
+ ATF_REQUIRE_EQ(fs::path("srcdir"), suite.source_root());
+ ATF_REQUIRE_EQ(fs::path("builddir"), suite.build_root());
+ ATF_REQUIRE_EQ(3, suite.test_programs().size());
+ ATF_REQUIRE_EQ(fs::path("builddir/one").to_absolute(),
+ suite.test_programs()[0]->absolute_path());
+ ATF_REQUIRE_EQ(fs::path("one"), suite.test_programs()[0]->relative_path());
+ ATF_REQUIRE_EQ("abc", suite.test_programs()[0]->test_suite_name());
+ ATF_REQUIRE_EQ(fs::path("builddir/dir/two").to_absolute(),
+ suite.test_programs()[1]->absolute_path());
+ ATF_REQUIRE_EQ(fs::path("dir/two"),
+ suite.test_programs()[1]->relative_path());
+ ATF_REQUIRE_EQ("def", suite.test_programs()[1]->test_suite_name());
+ ATF_REQUIRE_EQ(fs::path("builddir/dir/three").to_absolute(),
+ suite.test_programs()[2]->absolute_path());
+ ATF_REQUIRE_EQ(fs::path("dir/three"),
+ suite.test_programs()[2]->relative_path());
+ ATF_REQUIRE_EQ("foo", suite.test_programs()[2]->test_suite_name());
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__absolute_paths_are_stable);
+ATF_TEST_CASE_BODY(kyuafile__load__absolute_paths_are_stable)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "atf_test_program{name='one', test_suite='first'}\n");
+ atf::utils::create_file("one", "");
+
+ const engine::kyuafile suite = engine::kyuafile::load(
+ fs::path("config"), none, config::tree(), handle);
+
+ const fs::path previous_dir = fs::current_path();
+ fs::mkdir(fs::path("other"), 0755);
+ // Change the directory. We want later calls to absolute_path() on the test
+ // programs to return references to previous_dir instead.
+ ATF_REQUIRE(::chdir("other") != -1);
+
+ ATF_REQUIRE_EQ(fs::path("."), suite.source_root());
+ ATF_REQUIRE_EQ(fs::path("."), suite.build_root());
+ ATF_REQUIRE_EQ(1, suite.test_programs().size());
+ ATF_REQUIRE_EQ(previous_dir / "one",
+ suite.test_programs()[0]->absolute_path());
+ ATF_REQUIRE_EQ(fs::path("one"), suite.test_programs()[0]->relative_path());
+ ATF_REQUIRE_EQ("first", suite.test_programs()[0]->test_suite_name());
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__fs_calls_are_relative);
+ATF_TEST_CASE_BODY(kyuafile__load__fs_calls_are_relative)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ atf::utils::create_file(
+ "Kyuafile",
+ "syntax(2)\n"
+ "if fs.exists('one') then\n"
+ " plain_test_program{name='one', test_suite='first'}\n"
+ "end\n"
+ "if fs.exists('two') then\n"
+ " plain_test_program{name='two', test_suite='first'}\n"
+ "end\n"
+ "include('dir/Kyuafile')\n");
+ atf::utils::create_file("one", "");
+ fs::mkdir(fs::path("dir"), 0755);
+ atf::utils::create_file(
+ "dir/Kyuafile",
+ "syntax(2)\n"
+ "if fs.exists('one') then\n"
+ " plain_test_program{name='one', test_suite='first'}\n"
+ "end\n"
+ "if fs.exists('two') then\n"
+ " plain_test_program{name='two', test_suite='first'}\n"
+ "end\n");
+ atf::utils::create_file("dir/two", "");
+
+ const engine::kyuafile suite = engine::kyuafile::load(
+ fs::path("Kyuafile"), none, config::tree(), handle);
+
+ ATF_REQUIRE_EQ(2, suite.test_programs().size());
+ ATF_REQUIRE_EQ(fs::current_path() / "one",
+ suite.test_programs()[0]->absolute_path());
+ ATF_REQUIRE_EQ(fs::current_path() / "dir/two",
+ suite.test_programs()[1]->absolute_path());
+
+ handle.cleanup();
+}
+
+
+/// Verifies that load raises a load_error on a given input.
+///
+/// \param file Name of the file to load.
+/// \param regex Expression to match on load_error's contents.
+static void
+do_load_error_test(const char* file, const char* regex)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+ ATF_REQUIRE_THROW_RE(engine::load_error, regex,
+ engine::kyuafile::load(fs::path(file), none,
+ config::tree(), handle));
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__test_program_not_basename);
+ATF_TEST_CASE_BODY(kyuafile__load__test_program_not_basename)
+{
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "test_suite('abc')\n"
+ "atf_test_program{name='one'}\n"
+ "atf_test_program{name='./ls'}\n");
+
+ atf::utils::create_file("one", "");
+ do_load_error_test("config", "./ls.*path components");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__lua_error);
+ATF_TEST_CASE_BODY(kyuafile__load__lua_error)
+{
+ atf::utils::create_file("config", "this syntax is invalid\n");
+
+ do_load_error_test("config", ".*");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__syntax__not_called);
+ATF_TEST_CASE_BODY(kyuafile__load__syntax__not_called)
+{
+ atf::utils::create_file("config", "");
+
+ do_load_error_test("config", "syntax.* never called");
+}
+
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__syntax__deprecated_format);
+ATF_TEST_CASE_BODY(kyuafile__load__syntax__deprecated_format)
+{
+ atf::utils::create_file("config", "syntax('foo', 1)\n");
+ do_load_error_test("config", "must be 'kyuafile'");
+
+ atf::utils::create_file("config", "syntax('config', 2)\n");
+ do_load_error_test("config", "only takes one argument");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__syntax__twice);
+ATF_TEST_CASE_BODY(kyuafile__load__syntax__twice)
+{
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "syntax(2)\n");
+
+ do_load_error_test("config", "Can only call syntax.* once");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__syntax__bad_version);
+ATF_TEST_CASE_BODY(kyuafile__load__syntax__bad_version)
+{
+ atf::utils::create_file("config", "syntax(12)\n");
+
+ do_load_error_test("config", "Unsupported file version 12");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__test_suite__missing);
+ATF_TEST_CASE_BODY(kyuafile__load__test_suite__missing)
+{
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "plain_test_program{name='one'}");
+
+ atf::utils::create_file("one", "");
+
+ do_load_error_test("config", "No test suite defined");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__test_suite__twice);
+ATF_TEST_CASE_BODY(kyuafile__load__test_suite__twice)
+{
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "test_suite('foo')\n"
+ "test_suite('bar')\n");
+
+ do_load_error_test("config", "Can only call test_suite.* once");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__missing_file);
+ATF_TEST_CASE_BODY(kyuafile__load__missing_file)
+{
+ do_load_error_test("missing", "Load of 'missing' failed");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__missing_test_program);
+ATF_TEST_CASE_BODY(kyuafile__load__missing_test_program)
+{
+ atf::utils::create_file(
+ "config",
+ "syntax(2)\n"
+ "atf_test_program{name='one', test_suite='first'}\n"
+ "atf_test_program{name='two', test_suite='first'}\n");
+
+ atf::utils::create_file("one", "");
+
+ do_load_error_test("config", "Non-existent.*'two'");
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ scheduler::register_interface(
+ "atf", std::shared_ptr< scheduler::interface >(
+ new engine::atf_interface()));
+ scheduler::register_interface(
+ "plain", std::shared_ptr< scheduler::interface >(
+ new engine::plain_interface()));
+ scheduler::register_interface(
+ "tap", std::shared_ptr< scheduler::interface >(
+ new engine::tap_interface()));
+
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__empty);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__real_interfaces);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__mock_interfaces);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__metadata);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__current_directory);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__other_directory);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__build_directory);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__absolute_paths_are_stable);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__fs_calls_are_relative);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__test_program_not_basename);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__lua_error);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__syntax__not_called);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__syntax__deprecated_format);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__syntax__twice);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__syntax__bad_version);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__test_suite__missing);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__test_suite__twice);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__missing_file);
+ ATF_ADD_TEST_CASE(tcs, kyuafile__load__missing_test_program);
+}
diff --git a/engine/plain.cpp b/engine/plain.cpp
new file mode 100644
index 000000000000..8346e50bbecf
--- /dev/null
+++ b/engine/plain.cpp
@@ -0,0 +1,143 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/plain.hpp"
+
+extern "C" {
+#include <unistd.h>
+}
+
+#include <cstdlib>
+
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "model/test_result.hpp"
+#include "utils/defs.hpp"
+#include "utils/env.hpp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/optional.ipp"
+#include "utils/process/operations.hpp"
+#include "utils/process/status.hpp"
+#include "utils/sanity.hpp"
+
+namespace config = utils::config;
+namespace fs = utils::fs;
+namespace process = utils::process;
+
+using utils::optional;
+
+
+/// Executes a test program's list operation.
+///
+/// This method is intended to be called within a subprocess and is expected
+/// to terminate execution either by exec(2)ing the test program or by
+/// exiting with a failure.
+void
+engine::plain_interface::exec_list(
+ const model::test_program& /* test_program */,
+ const config::properties_map& /* vars */) const
+{
+ ::_exit(EXIT_SUCCESS);
+}
+
+
+/// Computes the test cases list of a test program.
+///
+/// \return A list of test cases.
+model::test_cases_map
+engine::plain_interface::parse_list(
+ const optional< process::status >& /* status */,
+ const fs::path& /* stdout_path */,
+ const fs::path& /* stderr_path */) const
+{
+ return model::test_cases_map_builder().add("main").build();
+}
+
+
+/// Executes a test case of the test program.
+///
+/// This method is intended to be called within a subprocess and is expected
+/// to terminate execution either by exec(2)ing the test program or by
+/// exiting with a failure.
+///
+/// \param test_program The test program to execute.
+/// \param test_case_name Name of the test case to invoke.
+/// \param vars User-provided variables to pass to the test program.
+void
+engine::plain_interface::exec_test(
+ const model::test_program& test_program,
+ const std::string& test_case_name,
+ const config::properties_map& vars,
+ const fs::path& /* control_directory */) const
+{
+ PRE(test_case_name == "main");
+
+ for (config::properties_map::const_iterator iter = vars.begin();
+ iter != vars.end(); ++iter) {
+ utils::setenv(F("TEST_ENV_%s") % (*iter).first, (*iter).second);
+ }
+
+ process::args_vector args;
+ process::exec(test_program.absolute_path(), args);
+}
+
+
+/// Computes the result of a test case based on its termination status.
+///
+/// \param status The termination status of the subprocess used to execute
+/// the exec_test() method or none if the test timed out.
+///
+/// \return A test result.
+model::test_result
+engine::plain_interface::compute_result(
+ const optional< process::status >& status,
+ const fs::path& /* control_directory */,
+ const fs::path& /* stdout_path */,
+ const fs::path& /* stderr_path */) const
+{
+ if (!status) {
+ return model::test_result(model::test_result_broken,
+ "Test case timed out");
+ }
+
+ if (status.get().exited()) {
+ const int exitstatus = status.get().exitstatus();
+ if (exitstatus == EXIT_SUCCESS) {
+ return model::test_result(model::test_result_passed);
+ } else {
+ return model::test_result(
+ model::test_result_failed,
+ F("Returned non-success exit status %s") % exitstatus);
+ }
+ } else {
+ return model::test_result(
+ model::test_result_broken,
+ F("Received signal %s") % status.get().termsig());
+ }
+}
diff --git a/engine/plain.hpp b/engine/plain.hpp
new file mode 100644
index 000000000000..ee5f3e746781
--- /dev/null
+++ b/engine/plain.hpp
@@ -0,0 +1,67 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/plain.hpp
+/// Execution engine for test programs that implement the plain interface.
+
+#if !defined(ENGINE_PLAIN_HPP)
+#define ENGINE_PLAIN_HPP
+
+#include "engine/scheduler.hpp"
+
+namespace engine {
+
+
+/// Implementation of the scheduler interface for plain test programs.
+class plain_interface : public engine::scheduler::interface {
+public:
+ void exec_list(const model::test_program&,
+ const utils::config::properties_map&) const UTILS_NORETURN;
+
+ model::test_cases_map parse_list(
+ const utils::optional< utils::process::status >&,
+ const utils::fs::path&,
+ const utils::fs::path&) const;
+
+ void exec_test(const model::test_program&, const std::string&,
+ const utils::config::properties_map&,
+ const utils::fs::path&) const
+ UTILS_NORETURN;
+
+ model::test_result compute_result(
+ const utils::optional< utils::process::status >&,
+ const utils::fs::path&,
+ const utils::fs::path&,
+ const utils::fs::path&) const;
+};
+
+
+} // namespace engine
+
+
+#endif // !defined(ENGINE_PLAIN_HPP)
diff --git a/engine/plain_helpers.cpp b/engine/plain_helpers.cpp
new file mode 100644
index 000000000000..52b1bc74fe10
--- /dev/null
+++ b/engine/plain_helpers.cpp
@@ -0,0 +1,238 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+extern "C" {
+#include <sys/stat.h>
+
+#include <unistd.h>
+
+extern char** environ;
+}
+
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+
+#include "utils/env.hpp"
+#include "utils/format/containers.ipp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/optional.ipp"
+#include "utils/test_utils.ipp"
+
+namespace fs = utils::fs;
+
+using utils::optional;
+
+
+namespace {
+
+
+/// Gets the name of the test case to run.
+///
+/// We use the value of the TEST_CASE environment variable if present, or
+/// else the basename of the test program.
+///
+/// \param arg0 Value of argv[0] as passed to main().
+///
+/// \return A test case name. The name may not be valid.
+static std::string
+guess_test_case_name(const char* arg0)
+{
+ const optional< std::string > test_case_env = utils::getenv("TEST_CASE");
+ if (test_case_env) {
+ return test_case_env.get();
+ } else {
+ return fs::path(arg0).leaf_name();
+ }
+}
+
+
+/// Logs an error message and exits the test with an error code.
+///
+/// \param str The error message to log.
+static void
+fail(const std::string& str)
+{
+ std::cerr << str << '\n';
+ std::exit(EXIT_FAILURE);
+}
+
+
+/// A test case that validates the TEST_ENV_* variables.
+static void
+test_check_configuration_variables(void)
+{
+ std::set< std::string > vars;
+ char** iter;
+ for (iter = environ; *iter != NULL; ++iter) {
+ if (std::strstr(*iter, "TEST_ENV_") == *iter) {
+ vars.insert(*iter);
+ }
+ }
+
+ std::set< std::string > exp_vars;
+ exp_vars.insert("TEST_ENV_first=some value");
+ exp_vars.insert("TEST_ENV_second=some other value");
+ if (vars != exp_vars) {
+ fail(F("Expected: %s\nFound: %s\n") % exp_vars % vars);
+ }
+}
+
+
+/// A test case that crashes.
+static void
+test_crash(void)
+{
+ utils::abort_without_coredump();
+}
+
+
+/// A test case that exits with a non-zero exit code, and not 1.
+static void
+test_fail(void)
+{
+ std::exit(8);
+}
+
+
+/// A test case that passes.
+static void
+test_pass(void)
+{
+}
+
+
+/// A test case that spawns a subchild that gets stuck.
+///
+/// This test case is used by the caller to validate that the whole process tree
+/// is terminated when the test case is killed.
+static void
+test_spawn_blocking_child(void)
+{
+ pid_t pid = ::fork();
+ if (pid == -1)
+ fail("Cannot fork subprocess");
+ else if (pid == 0) {
+ for (;;)
+ ::pause();
+ } else {
+ const fs::path name = fs::path(utils::getenv("CONTROL_DIR").get()) /
+ "pid";
+ std::ofstream pidfile(name.c_str());
+ if (!pidfile)
+ fail("Failed to create the pidfile");
+ pidfile << pid;
+ pidfile.close();
+ }
+}
+
+
+/// A test case that times out.
+///
+/// Note that the timeout is defined in the Kyuafile, as the plain interface has
+/// no means for test programs to specify this by themselves.
+static void
+test_timeout(void)
+{
+ ::sleep(10);
+ const fs::path control_dir = fs::path(utils::getenv("CONTROL_DIR").get());
+ std::ofstream file((control_dir / "cookie").c_str());
+ if (!file)
+ fail("Failed to create the control cookie");
+ file.close();
+}
+
+
+/// A test case that performs basic checks on the runtime environment.
+///
+/// If the runtime environment does not look clean (according to the rules in
+/// the Kyua runtime properties), the test fails.
+static void
+test_validate_isolation(void)
+{
+ if (utils::getenv("HOME").get() == "fake-value")
+ fail("HOME not reset");
+ if (utils::getenv("LANG"))
+ fail("LANG not unset");
+}
+
+
+} // anonymous namespace
+
+
+/// Entry point to the test program.
+///
+/// The caller can select which test case to run by defining the TEST_CASE
+/// environment variable. This is not "standard", in the sense this is not a
+/// generic property of the plain test case interface.
+///
+/// \todo It may be worth to split this binary into separate, smaller binaries,
+/// one for every "test case". We use this program as a dispatcher for
+/// different "main"s, the only reason being to keep the amount of helper test
+/// programs to a minimum. However, putting this each function in its own
+/// binary could simplify many other things.
+///
+/// \param argc The number of CLI arguments.
+/// \param argv The CLI arguments themselves. These are not used because
+/// Kyua will not pass any arguments to the plain test program.
+int
+main(int argc, char** argv)
+{
+ if (argc != 1) {
+ std::cerr << "No arguments allowed; select the test case with the "
+ "TEST_CASE variable";
+ return EXIT_FAILURE;
+ }
+
+ const std::string& test_case = guess_test_case_name(argv[0]);
+
+ if (test_case == "check_configuration_variables")
+ test_check_configuration_variables();
+ else if (test_case == "crash")
+ test_crash();
+ else if (test_case == "fail")
+ test_fail();
+ else if (test_case == "pass")
+ test_pass();
+ else if (test_case == "spawn_blocking_child")
+ test_spawn_blocking_child();
+ else if (test_case == "timeout")
+ test_timeout();
+ else if (test_case == "validate_isolation")
+ test_validate_isolation();
+ else {
+ std::cerr << "Unknown test case";
+ return EXIT_FAILURE;
+ }
+
+ return EXIT_SUCCESS;
+}
diff --git a/engine/plain_test.cpp b/engine/plain_test.cpp
new file mode 100644
index 000000000000..cc3326e4c581
--- /dev/null
+++ b/engine/plain_test.cpp
@@ -0,0 +1,207 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/plain.hpp"
+
+extern "C" {
+#include <signal.h>
+}
+
+#include <atf-c++.hpp>
+
+#include "engine/config.hpp"
+#include "engine/scheduler.hpp"
+#include "model/metadata.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "model/test_result.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/datetime.hpp"
+#include "utils/env.hpp"
+#include "utils/format/containers.ipp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/optional.ipp"
+
+namespace config = utils::config;
+namespace datetime = utils::datetime;
+namespace fs = utils::fs;
+namespace scheduler = engine::scheduler;
+
+using utils::none;
+
+
+namespace {
+
+
+/// Copies the plain helper to the work directory, selecting a specific helper.
+///
+/// \param tc Pointer to the calling test case, to obtain srcdir.
+/// \param name Name of the new binary to create. Must match the name of a
+/// valid helper, as the binary name is used to select it.
+static void
+copy_plain_helper(const atf::tests::tc* tc, const char* name)
+{
+ const fs::path srcdir(tc->get_config_var("srcdir"));
+ atf::utils::copy_file((srcdir / "plain_helpers").str(), name);
+}
+
+
+/// Runs one plain test program and checks its result.
+///
+/// \param tc Pointer to the calling test case, to obtain srcdir.
+/// \param test_case_name Name of the "test case" to select from the helper
+/// program.
+/// \param exp_result The expected result.
+/// \param metadata The test case metadata.
+/// \param user_config User-provided configuration variables.
+static void
+run_one(const atf::tests::tc* tc, const char* test_case_name,
+ const model::test_result& exp_result,
+ const model::metadata& metadata = model::metadata_builder().build(),
+ const config::tree& user_config = engine::empty_config())
+{
+ copy_plain_helper(tc, test_case_name);
+ const model::test_program_ptr program = model::test_program_builder(
+ "plain", fs::path(test_case_name), fs::current_path(), "the-suite")
+ .add_test_case("main", metadata).build_ptr();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+ (void)handle.spawn_test(program, "main", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: ");
+ atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: ");
+ ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result());
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+} // anonymous namespace
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list);
+ATF_TEST_CASE_BODY(list)
+{
+ const model::test_program program = model::test_program_builder(
+ "plain", fs::path("non-existent"), fs::path("."), "unused-suite")
+ .build();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+ const model::test_cases_map test_cases = handle.list_tests(
+ &program, engine::empty_config());
+ handle.cleanup();
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("main").build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__exit_success_is_pass);
+ATF_TEST_CASE_BODY(test__exit_success_is_pass)
+{
+ const model::test_result exp_result(model::test_result_passed);
+ run_one(this, "pass", exp_result);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__exit_non_zero_is_fail);
+ATF_TEST_CASE_BODY(test__exit_non_zero_is_fail)
+{
+ const model::test_result exp_result(
+ model::test_result_failed,
+ "Returned non-success exit status 8");
+ run_one(this, "fail", exp_result);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__signal_is_broken);
+ATF_TEST_CASE_BODY(test__signal_is_broken)
+{
+ const model::test_result exp_result(model::test_result_broken,
+ F("Received signal %s") % SIGABRT);
+ run_one(this, "crash", exp_result);
+}
+
+
+ATF_TEST_CASE(test__timeout_is_broken);
+ATF_TEST_CASE_HEAD(test__timeout_is_broken)
+{
+ set_md_var("timeout", "60");
+}
+ATF_TEST_CASE_BODY(test__timeout_is_broken)
+{
+ utils::setenv("CONTROL_DIR", fs::current_path().str());
+
+ const model::metadata metadata = model::metadata_builder()
+ .set_timeout(datetime::delta(1, 0)).build();
+ const model::test_result exp_result(model::test_result_broken,
+ "Test case timed out");
+ run_one(this, "timeout", exp_result, metadata);
+
+ ATF_REQUIRE(!atf::utils::file_exists("cookie"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__configuration_variables);
+ATF_TEST_CASE_BODY(test__configuration_variables)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.a-suite.first", "unused");
+ user_config.set_string("test_suites.the-suite.first", "some value");
+ user_config.set_string("test_suites.the-suite.second", "some other value");
+ user_config.set_string("test_suites.other-suite.first", "unused");
+
+ const model::test_result exp_result(model::test_result_passed);
+ run_one(this, "check_configuration_variables", exp_result,
+ model::metadata_builder().build(), user_config);
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ scheduler::register_interface(
+ "plain", std::shared_ptr< scheduler::interface >(
+ new engine::plain_interface()));
+
+ ATF_ADD_TEST_CASE(tcs, list);
+
+ ATF_ADD_TEST_CASE(tcs, test__exit_success_is_pass);
+ ATF_ADD_TEST_CASE(tcs, test__exit_non_zero_is_fail);
+ ATF_ADD_TEST_CASE(tcs, test__signal_is_broken);
+ ATF_ADD_TEST_CASE(tcs, test__timeout_is_broken);
+ ATF_ADD_TEST_CASE(tcs, test__configuration_variables);
+}
diff --git a/engine/requirements.cpp b/engine/requirements.cpp
new file mode 100644
index 000000000000..a7b0a90d97db
--- /dev/null
+++ b/engine/requirements.cpp
@@ -0,0 +1,293 @@
+// Copyright 2012 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/requirements.hpp"
+
+#include "model/metadata.hpp"
+#include "model/types.hpp"
+#include "utils/config/nodes.ipp"
+#include "utils/config/tree.ipp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/memory.hpp"
+#include "utils/passwd.hpp"
+#include "utils/sanity.hpp"
+#include "utils/units.hpp"
+
+namespace config = utils::config;
+namespace fs = utils::fs;
+namespace passwd = utils::passwd;
+namespace units = utils::units;
+
+
+namespace {
+
+
+/// Checks if all required configuration variables are present.
+///
+/// \param required_configs Set of required variable names.
+/// \param user_config Runtime user configuration.
+/// \param test_suite_name Name of the test suite the test belongs to.
+///
+/// \return Empty if all variables are present or an error message otherwise.
+static std::string
+check_required_configs(const model::strings_set& required_configs,
+ const config::tree& user_config,
+ const std::string& test_suite_name)
+{
+ for (model::strings_set::const_iterator iter = required_configs.begin();
+ iter != required_configs.end(); iter++) {
+ std::string property;
+ // TODO(jmmv): All this rewrite logic belongs in the ATF interface.
+ if ((*iter) == "unprivileged-user" || (*iter) == "unprivileged_user")
+ property = "unprivileged_user";
+ else
+ property = F("test_suites.%s.%s") % test_suite_name % (*iter);
+
+ if (!user_config.is_set(property))
+ return F("Required configuration property '%s' not defined") %
+ (*iter);
+ }
+ return "";
+}
+
+
+/// Checks if the allowed architectures match the current architecture.
+///
+/// \param allowed_architectures Set of allowed architectures.
+/// \param user_config Runtime user configuration.
+///
+/// \return Empty if the current architecture is in the list or an error
+/// message otherwise.
+static std::string
+check_allowed_architectures(const model::strings_set& allowed_architectures,
+ const config::tree& user_config)
+{
+ if (!allowed_architectures.empty()) {
+ const std::string architecture =
+ user_config.lookup< config::string_node >("architecture");
+ if (allowed_architectures.find(architecture) ==
+ allowed_architectures.end())
+ return F("Current architecture '%s' not supported") % architecture;
+ }
+ return "";
+}
+
+
+/// Checks if the allowed platforms match the current architecture.
+///
+/// \param allowed_platforms Set of allowed platforms.
+/// \param user_config Runtime user configuration.
+///
+/// \return Empty if the current platform is in the list or an error message
+/// otherwise.
+static std::string
+check_allowed_platforms(const model::strings_set& allowed_platforms,
+ const config::tree& user_config)
+{
+ if (!allowed_platforms.empty()) {
+ const std::string platform =
+ user_config.lookup< config::string_node >("platform");
+ if (allowed_platforms.find(platform) == allowed_platforms.end())
+ return F("Current platform '%s' not supported") % platform;
+ }
+ return "";
+}
+
+
+/// Checks if the current user matches the required user.
+///
+/// \param required_user Name of the required user category.
+/// \param user_config Runtime user configuration.
+///
+/// \return Empty if the current user fits the required user characteristics or
+/// an error message otherwise.
+static std::string
+check_required_user(const std::string& required_user,
+ const config::tree& user_config)
+{
+ if (!required_user.empty()) {
+ const passwd::user user = passwd::current_user();
+ if (required_user == "root") {
+ if (!user.is_root())
+ return "Requires root privileges";
+ } else if (required_user == "unprivileged") {
+ if (user.is_root())
+ if (!user_config.is_set("unprivileged_user"))
+ return "Requires an unprivileged user but the "
+ "unprivileged-user configuration variable is not "
+ "defined";
+ } else
+ UNREACHABLE_MSG("Value of require.user not properly validated");
+ }
+ return "";
+}
+
+
+/// Checks if all required files exist.
+///
+/// \param required_files Set of paths.
+///
+/// \return Empty if the required files all exist or an error message otherwise.
+static std::string
+check_required_files(const model::paths_set& required_files)
+{
+ for (model::paths_set::const_iterator iter = required_files.begin();
+ iter != required_files.end(); iter++) {
+ INV((*iter).is_absolute());
+ if (!fs::exists(*iter))
+ return F("Required file '%s' not found") % *iter;
+ }
+ return "";
+}
+
+
+/// Checks if all required programs exist.
+///
+/// \param required_programs Set of paths.
+///
+/// \return Empty if the required programs all exist or an error message
+/// otherwise.
+static std::string
+check_required_programs(const model::paths_set& required_programs)
+{
+ for (model::paths_set::const_iterator iter = required_programs.begin();
+ iter != required_programs.end(); iter++) {
+ if ((*iter).is_absolute()) {
+ if (!fs::exists(*iter))
+ return F("Required program '%s' not found") % *iter;
+ } else {
+ if (!fs::find_in_path((*iter).c_str()))
+ return F("Required program '%s' not found in PATH") % *iter;
+ }
+ }
+ return "";
+}
+
+
+/// Checks if the current system has the specified amount of memory.
+///
+/// \param required_memory Amount of required physical memory, or zero if not
+/// applicable.
+///
+/// \return Empty if the current system has the required amount of memory or an
+/// error message otherwise.
+static std::string
+check_required_memory(const units::bytes& required_memory)
+{
+ if (required_memory > 0) {
+ const units::bytes physical_memory = utils::physical_memory();
+ if (physical_memory > 0 && physical_memory < required_memory)
+ return F("Requires %s bytes of physical memory but only %s "
+ "available") %
+ required_memory.format() % physical_memory.format();
+ }
+ return "";
+}
+
+
+/// Checks if the work directory's file system has enough free disk space.
+///
+/// \param required_disk_space Amount of required free disk space, or zero if
+/// not applicable.
+/// \param work_directory Path to where the test case will be run.
+///
+/// \return Empty if the file system where the work directory is hosted has
+/// enough free disk space or an error message otherwise.
+static std::string
+check_required_disk_space(const units::bytes& required_disk_space,
+ const fs::path& work_directory)
+{
+ if (required_disk_space > 0) {
+ const units::bytes free_disk_space = fs::free_disk_space(
+ work_directory);
+ if (free_disk_space < required_disk_space)
+ return F("Requires %s bytes of free disk space but only %s "
+ "available") %
+ required_disk_space.format() % free_disk_space.format();
+ }
+ return "";
+}
+
+
+} // anonymous namespace
+
+
+/// Checks if all the requirements specified by the test case are met.
+///
+/// \param md The test metadata.
+/// \param cfg The engine configuration.
+/// \param test_suite Name of the test suite the test belongs to.
+/// \param work_directory Path to where the test case will be run.
+///
+/// \return A string describing the reason for skipping the test, or empty if
+/// the test should be executed.
+std::string
+engine::check_reqs(const model::metadata& md, const config::tree& cfg,
+ const std::string& test_suite,
+ const fs::path& work_directory)
+{
+ std::string reason;
+
+ reason = check_required_configs(md.required_configs(), cfg, test_suite);
+ if (!reason.empty())
+ return reason;
+
+ reason = check_allowed_architectures(md.allowed_architectures(), cfg);
+ if (!reason.empty())
+ return reason;
+
+ reason = check_allowed_platforms(md.allowed_platforms(), cfg);
+ if (!reason.empty())
+ return reason;
+
+ reason = check_required_user(md.required_user(), cfg);
+ if (!reason.empty())
+ return reason;
+
+ reason = check_required_files(md.required_files());
+ if (!reason.empty())
+ return reason;
+
+ reason = check_required_programs(md.required_programs());
+ if (!reason.empty())
+ return reason;
+
+ reason = check_required_memory(md.required_memory());
+ if (!reason.empty())
+ return reason;
+
+ reason = check_required_disk_space(md.required_disk_space(),
+ work_directory);
+ if (!reason.empty())
+ return reason;
+
+ INV(reason.empty());
+ return reason;
+}
diff --git a/engine/requirements.hpp b/engine/requirements.hpp
new file mode 100644
index 000000000000..a36a938b3034
--- /dev/null
+++ b/engine/requirements.hpp
@@ -0,0 +1,51 @@
+// Copyright 2012 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/requirements.hpp
+/// Handling of test case requirements.
+
+#if !defined(ENGINE_REQUIREMENTS_HPP)
+#define ENGINE_REQUIREMENTS_HPP
+
+#include <string>
+
+#include "model/metadata_fwd.hpp"
+#include "utils/config/tree_fwd.hpp"
+#include "utils/fs/path_fwd.hpp"
+
+namespace engine {
+
+
+std::string check_reqs(const model::metadata&, const utils::config::tree&,
+ const std::string&, const utils::fs::path&);
+
+
+} // namespace engine
+
+
+#endif // !defined(ENGINE_REQUIREMENTS_HPP)
diff --git a/engine/requirements_test.cpp b/engine/requirements_test.cpp
new file mode 100644
index 000000000000..5052da932cb6
--- /dev/null
+++ b/engine/requirements_test.cpp
@@ -0,0 +1,511 @@
+// Copyright 2012 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "model/metadata.hpp"
+
+#include <atf-c++.hpp>
+
+#include "engine/config.hpp"
+#include "engine/requirements.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/env.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/memory.hpp"
+#include "utils/passwd.hpp"
+#include "utils/units.hpp"
+
+namespace config = utils::config;
+namespace fs = utils::fs;
+namespace passwd = utils::passwd;
+namespace units = utils::units;
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__none);
+ATF_TEST_CASE_BODY(check_reqs__none)
+{
+ const model::metadata md = model::metadata_builder().build();
+ ATF_REQUIRE(engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_architectures__one_ok);
+ATF_TEST_CASE_BODY(check_reqs__allowed_architectures__one_ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_allowed_architecture("x86_64")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("architecture", "x86_64");
+ user_config.set_string("platform", "");
+ ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_architectures__one_fail);
+ATF_TEST_CASE_BODY(check_reqs__allowed_architectures__one_fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_allowed_architecture("x86_64")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("architecture", "i386");
+ user_config.set_string("platform", "");
+ ATF_REQUIRE_MATCH("Current architecture 'i386' not supported",
+ engine::check_reqs(md, user_config, "", fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_architectures__many_ok);
+ATF_TEST_CASE_BODY(check_reqs__allowed_architectures__many_ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_allowed_architecture("x86_64")
+ .add_allowed_architecture("i386")
+ .add_allowed_architecture("powerpc")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("architecture", "i386");
+ user_config.set_string("platform", "");
+ ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_architectures__many_fail);
+ATF_TEST_CASE_BODY(check_reqs__allowed_architectures__many_fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_allowed_architecture("x86_64")
+ .add_allowed_architecture("i386")
+ .add_allowed_architecture("powerpc")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("architecture", "arm");
+ user_config.set_string("platform", "");
+ ATF_REQUIRE_MATCH("Current architecture 'arm' not supported",
+ engine::check_reqs(md, user_config, "", fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_platforms__one_ok);
+ATF_TEST_CASE_BODY(check_reqs__allowed_platforms__one_ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_allowed_platform("amd64")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("architecture", "");
+ user_config.set_string("platform", "amd64");
+ ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_platforms__one_fail);
+ATF_TEST_CASE_BODY(check_reqs__allowed_platforms__one_fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_allowed_platform("amd64")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("architecture", "");
+ user_config.set_string("platform", "i386");
+ ATF_REQUIRE_MATCH("Current platform 'i386' not supported",
+ engine::check_reqs(md, user_config, "", fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_platforms__many_ok);
+ATF_TEST_CASE_BODY(check_reqs__allowed_platforms__many_ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_allowed_platform("amd64")
+ .add_allowed_platform("i386")
+ .add_allowed_platform("macppc")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("architecture", "");
+ user_config.set_string("platform", "i386");
+ ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_platforms__many_fail);
+ATF_TEST_CASE_BODY(check_reqs__allowed_platforms__many_fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_allowed_platform("amd64")
+ .add_allowed_platform("i386")
+ .add_allowed_platform("macppc")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("architecture", "");
+ user_config.set_string("platform", "shark");
+ ATF_REQUIRE_MATCH("Current platform 'shark' not supported",
+ engine::check_reqs(md, user_config, "", fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_configs__one_ok);
+ATF_TEST_CASE_BODY(check_reqs__required_configs__one_ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_required_config("my-var")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("test_suites.suite.aaa", "value1");
+ user_config.set_string("test_suites.suite.my-var", "value2");
+ user_config.set_string("test_suites.suite.zzz", "value3");
+ ATF_REQUIRE(engine::check_reqs(md, user_config, "suite",
+ fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_configs__one_fail);
+ATF_TEST_CASE_BODY(check_reqs__required_configs__one_fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_required_config("unprivileged_user")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("test_suites.suite.aaa", "value1");
+ user_config.set_string("test_suites.suite.my-var", "value2");
+ user_config.set_string("test_suites.suite.zzz", "value3");
+ ATF_REQUIRE_MATCH("Required configuration property 'unprivileged_user' not "
+ "defined",
+ engine::check_reqs(md, user_config, "suite",
+ fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_configs__many_ok);
+ATF_TEST_CASE_BODY(check_reqs__required_configs__many_ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_required_config("foo")
+ .add_required_config("bar")
+ .add_required_config("baz")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("test_suites.suite.aaa", "value1");
+ user_config.set_string("test_suites.suite.foo", "value2");
+ user_config.set_string("test_suites.suite.bar", "value3");
+ user_config.set_string("test_suites.suite.baz", "value4");
+ user_config.set_string("test_suites.suite.zzz", "value5");
+ ATF_REQUIRE(engine::check_reqs(md, user_config, "suite",
+ fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_configs__many_fail);
+ATF_TEST_CASE_BODY(check_reqs__required_configs__many_fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_required_config("foo")
+ .add_required_config("bar")
+ .add_required_config("baz")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set_string("test_suites.suite.aaa", "value1");
+ user_config.set_string("test_suites.suite.foo", "value2");
+ user_config.set_string("test_suites.suite.zzz", "value3");
+ ATF_REQUIRE_MATCH("Required configuration property 'bar' not defined",
+ engine::check_reqs(md, user_config, "suite",
+ fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_configs__special);
+ATF_TEST_CASE_BODY(check_reqs__required_configs__special)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_required_config("unprivileged-user")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ ATF_REQUIRE_MATCH("Required configuration property 'unprivileged-user' "
+ "not defined",
+ engine::check_reqs(md, user_config, "", fs::path(".")));
+ user_config.set< engine::user_node >(
+ "unprivileged_user", passwd::user("foo", 1, 2));
+ ATF_REQUIRE(engine::check_reqs(md, user_config, "foo",
+ fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_user__root__ok);
+ATF_TEST_CASE_BODY(check_reqs__required_user__root__ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .set_required_user("root")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ ATF_REQUIRE(!user_config.is_set("unprivileged_user"));
+
+ passwd::set_current_user_for_testing(passwd::user("", 0, 1));
+ ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_user__root__fail);
+ATF_TEST_CASE_BODY(check_reqs__required_user__root__fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .set_required_user("root")
+ .build();
+
+ passwd::set_current_user_for_testing(passwd::user("", 123, 1));
+ ATF_REQUIRE_MATCH("Requires root privileges",
+ engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_user__unprivileged__same);
+ATF_TEST_CASE_BODY(check_reqs__required_user__unprivileged__same)
+{
+ const model::metadata md = model::metadata_builder()
+ .set_required_user("unprivileged")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ ATF_REQUIRE(!user_config.is_set("unprivileged_user"));
+
+ passwd::set_current_user_for_testing(passwd::user("", 123, 1));
+ ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_user__unprivileged__ok);
+ATF_TEST_CASE_BODY(check_reqs__required_user__unprivileged__ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .set_required_user("unprivileged")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ user_config.set< engine::user_node >(
+ "unprivileged_user", passwd::user("", 123, 1));
+
+ passwd::set_current_user_for_testing(passwd::user("", 0, 1));
+ ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_user__unprivileged__fail);
+ATF_TEST_CASE_BODY(check_reqs__required_user__unprivileged__fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .set_required_user("unprivileged")
+ .build();
+
+ config::tree user_config = engine::default_config();
+ ATF_REQUIRE(!user_config.is_set("unprivileged_user"));
+
+ passwd::set_current_user_for_testing(passwd::user("", 0, 1));
+ ATF_REQUIRE_MATCH("Requires.*unprivileged.*unprivileged-user",
+ engine::check_reqs(md, user_config, "", fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_disk_space__ok);
+ATF_TEST_CASE_BODY(check_reqs__required_disk_space__ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .set_required_disk_space(units::bytes::parse("1m"))
+ .build();
+
+ ATF_REQUIRE(engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_disk_space__fail);
+ATF_TEST_CASE_BODY(check_reqs__required_disk_space__fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .set_required_disk_space(units::bytes::parse("1000t"))
+ .build();
+
+ ATF_REQUIRE_MATCH("Requires 1000.00T .*disk space",
+ engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_files__ok);
+ATF_TEST_CASE_BODY(check_reqs__required_files__ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_required_file(fs::current_path() / "test-file")
+ .build();
+
+ atf::utils::create_file("test-file", "");
+
+ ATF_REQUIRE(engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_files__fail);
+ATF_TEST_CASE_BODY(check_reqs__required_files__fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_required_file(fs::path("/non-existent/file"))
+ .build();
+
+ ATF_REQUIRE_MATCH("'/non-existent/file' not found$",
+ engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_memory__ok);
+ATF_TEST_CASE_BODY(check_reqs__required_memory__ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .set_required_memory(units::bytes::parse("1m"))
+ .build();
+
+ ATF_REQUIRE(engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_memory__fail);
+ATF_TEST_CASE_BODY(check_reqs__required_memory__fail)
+{
+ const model::metadata md = model::metadata_builder()
+ .set_required_memory(units::bytes::parse("100t"))
+ .build();
+
+ if (utils::physical_memory() == 0)
+ skip("Don't know how to query the amount of physical memory");
+ ATF_REQUIRE_MATCH("Requires 100.00T .*memory",
+ engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")));
+}
+
+
+ATF_TEST_CASE(check_reqs__required_programs__ok);
+ATF_TEST_CASE_HEAD(check_reqs__required_programs__ok)
+{
+ set_md_var("require.progs", "/bin/ls /bin/mv");
+}
+ATF_TEST_CASE_BODY(check_reqs__required_programs__ok)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_required_program(fs::path("/bin/ls"))
+ .add_required_program(fs::path("foo"))
+ .add_required_program(fs::path("/bin/mv"))
+ .build();
+
+ fs::mkdir(fs::path("bin"), 0755);
+ atf::utils::create_file("bin/foo", "");
+ utils::setenv("PATH", (fs::current_path() / "bin").str());
+
+ ATF_REQUIRE(engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")).empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_programs__fail_absolute);
+ATF_TEST_CASE_BODY(check_reqs__required_programs__fail_absolute)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_required_program(fs::path("/non-existent/program"))
+ .build();
+
+ ATF_REQUIRE_MATCH("'/non-existent/program' not found$",
+ engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_programs__fail_relative);
+ATF_TEST_CASE_BODY(check_reqs__required_programs__fail_relative)
+{
+ const model::metadata md = model::metadata_builder()
+ .add_required_program(fs::path("foo"))
+ .add_required_program(fs::path("bar"))
+ .build();
+
+ fs::mkdir(fs::path("bin"), 0755);
+ atf::utils::create_file("bin/foo", "");
+ utils::setenv("PATH", (fs::current_path() / "bin").str());
+
+ ATF_REQUIRE_MATCH("'bar' not found in PATH$",
+ engine::check_reqs(md, engine::empty_config(), "",
+ fs::path(".")));
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, check_reqs__none);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_architectures__one_ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_architectures__one_fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_architectures__many_ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_architectures__many_fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_platforms__one_ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_platforms__one_fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_platforms__many_ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_platforms__many_fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_configs__one_ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_configs__one_fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_configs__many_ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_configs__many_fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_configs__special);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_user__root__ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_user__root__fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_user__unprivileged__same);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_user__unprivileged__ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_user__unprivileged__fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_disk_space__ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_disk_space__fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_files__ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_files__fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_memory__ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_memory__fail);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_programs__ok);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_programs__fail_absolute);
+ ATF_ADD_TEST_CASE(tcs, check_reqs__required_programs__fail_relative);
+}
diff --git a/engine/scanner.cpp b/engine/scanner.cpp
new file mode 100644
index 000000000000..b42b089c3c3c
--- /dev/null
+++ b/engine/scanner.cpp
@@ -0,0 +1,216 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/scanner.hpp"
+
+#include <deque>
+#include <string>
+
+#include "engine/filters.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "utils/noncopyable.hpp"
+#include "utils/optional.ipp"
+#include "utils/sanity.hpp"
+
+using utils::none;
+using utils::optional;
+
+
+namespace {
+
+
+/// Extracts the keys of a map as a deque.
+///
+/// \tparam KeyType The type of the map keys.
+/// \tparam ValueType The type of the map values.
+/// \param map The input map.
+///
+/// \return A deque with the keys of the map.
+template< typename KeyType, typename ValueType >
+static std::deque< KeyType >
+map_keys(const std::map< KeyType, ValueType >& map)
+{
+ std::deque< KeyType > keys;
+ for (typename std::map< KeyType, ValueType >::const_iterator iter =
+ map.begin(); iter != map.end(); ++iter) {
+ keys.push_back((*iter).first);
+ }
+ return keys;
+}
+
+
+} // anonymous namespace
+
+
+/// Internal implementation for the scanner class.
+struct engine::scanner::impl : utils::noncopyable {
+ /// Collection of test programs not yet scanned.
+ ///
+ /// The first element in this deque is the "active" test program when
+ /// first_test_cases is defined.
+ std::deque< model::test_program_ptr > pending_test_programs;
+
+ /// Current state of the provided filters.
+ engine::filters_state filters;
+
+ /// Collection of test cases not yet scanned.
+ ///
+ /// These are the test cases for the first test program in
+ /// pending_test_programs when such test program is active.
+ optional< std::deque< std::string > > first_test_cases;
+
+ /// Constructor.
+ ///
+ /// \param test_programs_ Collection of test programs to scan through.
+ /// \param filters_ List of scan filters as provided by the user.
+ impl(const model::test_programs_vector& test_programs_,
+ const std::set< engine::test_filter >& filters_) :
+ pending_test_programs(test_programs_.begin(), test_programs_.end()),
+ filters(filters_)
+ {
+ }
+
+ /// Positions the internal state to return the next element if any.
+ ///
+ /// \post If there are more elements to read, returns true and
+ /// pending_test_programs[0] points to the active test program and
+ /// first_test_cases[0] has the test case to be returned.
+ ///
+ /// \return True if there is one more result available.
+ bool
+ advance(void)
+ {
+ for (;;) {
+ if (first_test_cases) {
+ if (first_test_cases.get().empty()) {
+ pending_test_programs.pop_front();
+ first_test_cases = none;
+ }
+ }
+ if (pending_test_programs.empty()) {
+ break;
+ }
+
+ model::test_program_ptr test_program = pending_test_programs[0];
+ if (!first_test_cases) {
+ if (!filters.match_test_program(
+ test_program->relative_path())) {
+ pending_test_programs.pop_front();
+ continue;
+ }
+
+ first_test_cases = utils::make_optional(
+ map_keys(test_program->test_cases()));
+ }
+
+ if (!first_test_cases.get().empty()) {
+ std::deque< std::string >::iterator iter =
+ first_test_cases.get().begin();
+ const std::string test_case_name = *iter;
+ if (!filters.match_test_case(test_program->relative_path(),
+ test_case_name)) {
+ first_test_cases.get().erase(iter);
+ continue;
+ }
+ return true;
+ } else {
+ pending_test_programs.pop_front();
+ first_test_cases = none;
+ }
+ }
+ return false;
+ }
+
+ /// Extracts the current element.
+ ///
+ /// \pre Must be called only if advance() returns true, and immediately
+ /// afterwards.
+ ///
+ /// \return The current scan result.
+ engine::scan_result
+ consume(void)
+ {
+ const std::string test_case_name = first_test_cases.get()[0];
+ first_test_cases.get().pop_front();
+ return scan_result(pending_test_programs[0], test_case_name);
+ }
+};
+
+
+/// Constructor.
+///
+/// \param test_programs Collection of test programs to scan through.
+/// \param filters List of scan filters as provided by the user.
+engine::scanner::scanner(const model::test_programs_vector& test_programs,
+ const std::set< engine::test_filter >& filters) :
+ _pimpl(new impl(test_programs, filters))
+{
+}
+
+
+/// Destructor.
+engine::scanner::~scanner(void)
+{
+}
+
+
+/// Returns the next scan result.
+///
+/// \return A scan result if there are still pending test cases to be processed,
+/// or none otherwise.
+optional< engine::scan_result >
+engine::scanner::yield(void)
+{
+ if (_pimpl->advance()) {
+ return utils::make_optional(_pimpl->consume());
+ } else {
+ return none;
+ }
+}
+
+
+/// Checks whether the scan is finished.
+///
+/// \return True if the scan is finished, in which case yield() will return
+/// none; false otherwise.
+bool
+engine::scanner::done(void)
+{
+ return !_pimpl->advance();
+}
+
+
+/// Returns the list of test filters that did not match any test case.
+///
+/// \return The collection of unmatched test filters.
+std::set< engine::test_filter >
+engine::scanner::unused_filters(void) const
+{
+ return _pimpl->filters.unused();
+}
diff --git a/engine/scanner.hpp b/engine/scanner.hpp
new file mode 100644
index 000000000000..722bc9be5f4c
--- /dev/null
+++ b/engine/scanner.hpp
@@ -0,0 +1,76 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/scanner.hpp
+/// Utilities to scan through list of tests in a test suite.
+
+#if !defined(ENGINE_SCANNER_HPP)
+#define ENGINE_SCANNER_HPP
+
+#include "engine/scanner_fwd.hpp"
+
+#include <memory>
+#include <set>
+
+#include "engine/filters_fwd.hpp"
+#include "model/test_program_fwd.hpp"
+#include "utils/optional_fwd.hpp"
+
+namespace engine {
+
+
+/// Scans a list of test programs, yielding one test case at a time.
+///
+/// This class contains the state necessary to process a collection of test
+/// programs (possibly as provided by the Kyuafile) and to extract an arbitrary
+/// (test program, test_case) pair out of them one at a time.
+///
+/// The scanning algorithm guarantees that test programs are initialized
+/// dynamically, should they need to load their list of test cases from disk.
+///
+/// The order of the extraction is not guaranteed.
+class scanner {
+ struct impl;
+ /// Pointer to the internal implementation data.
+ std::shared_ptr< impl > _pimpl;
+
+public:
+ scanner(const model::test_programs_vector&, const std::set< test_filter >&);
+ ~scanner(void);
+
+ bool done(void);
+ utils::optional< scan_result > yield(void);
+
+ std::set< test_filter > unused_filters(void) const;
+};
+
+
+} // namespace engine
+
+
+#endif // !defined(ENGINE_SCANNER_HPP)
diff --git a/engine/scanner_fwd.hpp b/engine/scanner_fwd.hpp
new file mode 100644
index 000000000000..5c91888fa266
--- /dev/null
+++ b/engine/scanner_fwd.hpp
@@ -0,0 +1,59 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/scanner_fwd.hpp
+/// Forward declarations for engine/scanner.hpp
+
+#if !defined(ENGINE_SCANNER_FWD_HPP)
+#define ENGINE_SCANNER_FWD_HPP
+
+#include <string>
+#include <utility>
+
+#include "model/test_program_fwd.hpp"
+
+namespace engine {
+
+
+/// Result type yielded by the scanner: a (test program, test case name) pair.
+///
+/// We must use model::test_program_ptr here instead of model::test_program
+/// because we must keep the polimorphic properties of the test program. In
+/// particular, if the test program comes from the Kyuafile and is of the type
+/// model::lazy_test_program, we must keep access to the loaded list of test
+/// cases (which, for obscure reasons, is kept in the subclass).
+/// TODO(jmmv): This is ugly, very ugly. There has to be a better way.
+typedef std::pair< model::test_program_ptr, std::string > scan_result;
+
+
+class scanner;
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_SCANNER_FWD_HPP)
diff --git a/engine/scanner_test.cpp b/engine/scanner_test.cpp
new file mode 100644
index 000000000000..f79717eca49e
--- /dev/null
+++ b/engine/scanner_test.cpp
@@ -0,0 +1,476 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/scanner.hpp"
+
+#include <cstdarg>
+#include <cstddef>
+#include <typeinfo>
+
+#include <atf-c++.hpp>
+
+#include "engine/filters.hpp"
+#include "model/metadata.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "utils/format/containers.ipp"
+#include "utils/fs/path.hpp"
+#include "utils/optional.ipp"
+
+namespace fs = utils::fs;
+
+using utils::optional;
+
+
+namespace {
+
+
+/// Test program that implements a mock test_cases() lazy call.
+class mock_test_program : public model::test_program {
+ /// Number of times test_cases has been called.
+ mutable std::size_t _num_calls;
+
+ /// Collection of test cases; lazily initialized.
+ mutable model::test_cases_map _test_cases;
+
+public:
+ /// Constructs a new test program.
+ ///
+ /// \param binary_ The name of the test program binary relative to root_.
+ mock_test_program(const fs::path& binary_) :
+ test_program("unused-interface", binary_, fs::path("unused-root"),
+ "unused-suite", model::metadata_builder().build(),
+ model::test_cases_map()),
+ _num_calls(0)
+ {
+ }
+
+ /// Gets or loads the list of test cases from the test program.
+ ///
+ /// \return The list of test cases provided by the test program.
+ const model::test_cases_map&
+ test_cases(void) const
+ {
+ if (_num_calls == 0) {
+ const model::metadata metadata = model::metadata_builder().build();
+ const model::test_case tc1("one", metadata);
+ const model::test_case tc2("two", metadata);
+ _test_cases.insert(model::test_cases_map::value_type("one", tc1));
+ _test_cases.insert(model::test_cases_map::value_type("two", tc2));
+ }
+ _num_calls++;
+ return _test_cases;
+ }
+
+ /// Returns the number of times test_cases() has been called.
+ ///
+ /// \return A counter.
+ std::size_t
+ num_calls(void) const
+ {
+ return _num_calls;
+ }
+};
+
+
+/// Syntactic sugar to instantiate a test program with various test cases.
+///
+/// The scanner only cares about the relative path of the test program object
+/// and the names of the test cases. This function helps in instantiating a
+/// test program that has the minimum set of details only.
+///
+/// \param relative_path Relative path to the test program.
+/// \param ... List of test case names to add to the test program. Must be
+/// NULL-terminated.
+///
+/// \return A constructed test program.
+static model::test_program_ptr
+new_test_program(const char* relative_path, ...)
+{
+ model::test_program_builder builder(
+ "unused-interface", fs::path(relative_path), fs::path("unused-root"),
+ "unused-suite");
+
+ va_list ap;
+ va_start(ap, relative_path);
+ const char* test_case_name;
+ while ((test_case_name = va_arg(ap, const char*)) != NULL) {
+ builder.add_test_case(test_case_name);
+ }
+ va_end(ap);
+
+ return builder.build_ptr();
+}
+
+
+/// Yields all test cases in the scanner for simplicity of testing.
+///
+/// In most of the tests below, we just care about the scanner returning the
+/// full set of matching test cases, not the specific behavior of every single
+/// yield() call. This function just returns the whole set, which helps in
+/// writing functional tests.
+///
+/// \param scanner The scanner on which to iterate.
+///
+/// \return The full collection of results yielded by the scanner.
+static std::set< engine::scan_result >
+yield_all(engine::scanner& scanner)
+{
+ std::set< engine::scan_result > results;
+ while (!scanner.done()) {
+ const optional< engine::scan_result > result = scanner.yield();
+ ATF_REQUIRE(result);
+ results.insert(result.get());
+ }
+ ATF_REQUIRE(!scanner.yield());
+ ATF_REQUIRE(scanner.done());
+ return results;
+}
+
+
+} // anonymous namespace
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__no_tests);
+ATF_TEST_CASE_BODY(scanner__no_filters__no_tests)
+{
+ const model::test_programs_vector test_programs;
+ const std::set< engine::test_filter > filters;
+
+ engine::scanner scanner(test_programs, filters);
+ ATF_REQUIRE(scanner.done());
+ ATF_REQUIRE(!scanner.yield());
+ ATF_REQUIRE(scanner.unused_filters().empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__one_test_in_one_program);
+ATF_TEST_CASE_BODY(scanner__no_filters__one_test_in_one_program)
+{
+ const model::test_program_ptr test_program = new_test_program(
+ "dir/program", "lone_test", NULL);
+
+ model::test_programs_vector test_programs;
+ test_programs.push_back(test_program);
+
+ const std::set< engine::test_filter > filters;
+
+ std::set< engine::scan_result > exp_results;
+ exp_results.insert(engine::scan_result(test_program, "lone_test"));
+
+ engine::scanner scanner(test_programs, filters);
+ const std::set< engine::scan_result > results = yield_all(scanner);
+ ATF_REQUIRE_EQ(exp_results, results);
+ ATF_REQUIRE(scanner.unused_filters().empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__one_test_per_many_programs);
+ATF_TEST_CASE_BODY(scanner__no_filters__one_test_per_many_programs)
+{
+ const model::test_program_ptr test_program1 = new_test_program(
+ "dir/program1", "foo_test", NULL);
+ const model::test_program_ptr test_program2 = new_test_program(
+ "program2", "bar_test", NULL);
+ const model::test_program_ptr test_program3 = new_test_program(
+ "a/b/c/d/e/program3", "baz_test", NULL);
+
+ model::test_programs_vector test_programs;
+ test_programs.push_back(test_program1);
+ test_programs.push_back(test_program2);
+ test_programs.push_back(test_program3);
+
+ const std::set< engine::test_filter > filters;
+
+ std::set< engine::scan_result > exp_results;
+ exp_results.insert(engine::scan_result(test_program1, "foo_test"));
+ exp_results.insert(engine::scan_result(test_program2, "bar_test"));
+ exp_results.insert(engine::scan_result(test_program3, "baz_test"));
+
+ engine::scanner scanner(test_programs, filters);
+ const std::set< engine::scan_result > results = yield_all(scanner);
+ ATF_REQUIRE_EQ(exp_results, results);
+ ATF_REQUIRE(scanner.unused_filters().empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__many_tests_in_one_program);
+ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_in_one_program)
+{
+ const model::test_program_ptr test_program = new_test_program(
+ "dir/program", "first_test", "second_test", "third_test", NULL);
+
+ model::test_programs_vector test_programs;
+ test_programs.push_back(test_program);
+
+ const std::set< engine::test_filter > filters;
+
+ std::set< engine::scan_result > exp_results;
+ exp_results.insert(engine::scan_result(test_program, "first_test"));
+ exp_results.insert(engine::scan_result(test_program, "second_test"));
+ exp_results.insert(engine::scan_result(test_program, "third_test"));
+
+ engine::scanner scanner(test_programs, filters);
+ const std::set< engine::scan_result > results = yield_all(scanner);
+ ATF_REQUIRE_EQ(exp_results, results);
+ ATF_REQUIRE(scanner.unused_filters().empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__many_tests_per_many_programs);
+ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_per_many_programs)
+{
+ const model::test_program_ptr test_program1 = new_test_program(
+ "dir/program1", "foo_test", "bar_test", "baz_test", NULL);
+ const model::test_program_ptr test_program2 = new_test_program(
+ "program2", "lone_test", NULL);
+ const model::test_program_ptr test_program3 = new_test_program(
+ "a/b/c/d/e/program3", "another_test", "last_test", NULL);
+
+ model::test_programs_vector test_programs;
+ test_programs.push_back(test_program1);
+ test_programs.push_back(test_program2);
+ test_programs.push_back(test_program3);
+
+ const std::set< engine::test_filter > filters;
+
+ std::set< engine::scan_result > exp_results;
+ exp_results.insert(engine::scan_result(test_program1, "foo_test"));
+ exp_results.insert(engine::scan_result(test_program1, "bar_test"));
+ exp_results.insert(engine::scan_result(test_program1, "baz_test"));
+ exp_results.insert(engine::scan_result(test_program2, "lone_test"));
+ exp_results.insert(engine::scan_result(test_program3, "another_test"));
+ exp_results.insert(engine::scan_result(test_program3, "last_test"));
+
+ engine::scanner scanner(test_programs, filters);
+ const std::set< engine::scan_result > results = yield_all(scanner);
+ ATF_REQUIRE_EQ(exp_results, results);
+ ATF_REQUIRE(scanner.unused_filters().empty());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__verify_lazy_loads);
+ATF_TEST_CASE_BODY(scanner__no_filters__verify_lazy_loads)
+{
+ const model::test_program_ptr test_program1(new mock_test_program(
+ fs::path("first")));
+ const mock_test_program* mock_program1 =
+ dynamic_cast< const mock_test_program* >(test_program1.get());
+ const model::test_program_ptr test_program2(new mock_test_program(
+ fs::path("second")));
+ const mock_test_program* mock_program2 =
+ dynamic_cast< const mock_test_program* >(test_program2.get());
+
+ model::test_programs_vector test_programs;
+ test_programs.push_back(test_program1);
+ test_programs.push_back(test_program2);
+
+ const std::set< engine::test_filter > filters;
+
+ std::set< engine::scan_result > exp_results;
+ exp_results.insert(engine::scan_result(test_program1, "one"));
+ exp_results.insert(engine::scan_result(test_program1, "two"));
+ exp_results.insert(engine::scan_result(test_program2, "one"));
+ exp_results.insert(engine::scan_result(test_program2, "two"));
+
+ engine::scanner scanner(test_programs, filters);
+ std::set< engine::scan_result > results;
+ ATF_REQUIRE_EQ(0, mock_program1->num_calls());
+ ATF_REQUIRE_EQ(0, mock_program2->num_calls());
+
+ // This abuses the internal implementation of the scanner by making
+ // assumptions on the order of the results.
+ results.insert(scanner.yield().get());
+ ATF_REQUIRE_EQ(1, mock_program1->num_calls());
+ ATF_REQUIRE_EQ(0, mock_program2->num_calls());
+ results.insert(scanner.yield().get());
+ ATF_REQUIRE_EQ(1, mock_program1->num_calls());
+ ATF_REQUIRE_EQ(0, mock_program2->num_calls());
+ results.insert(scanner.yield().get());
+ ATF_REQUIRE_EQ(1, mock_program1->num_calls());
+ ATF_REQUIRE_EQ(1, mock_program2->num_calls());
+ results.insert(scanner.yield().get());
+ ATF_REQUIRE_EQ(1, mock_program1->num_calls());
+ ATF_REQUIRE_EQ(1, mock_program2->num_calls());
+ ATF_REQUIRE(scanner.done());
+
+ ATF_REQUIRE_EQ(exp_results, results);
+ ATF_REQUIRE(scanner.unused_filters().empty());
+
+ // Make sure we are still talking to the original objects.
+ for (std::set< engine::scan_result >::const_iterator iter = results.begin();
+ iter != results.end(); ++iter) {
+ const mock_test_program* mock_program =
+ dynamic_cast< const mock_test_program* >((*iter).first.get());
+ ATF_REQUIRE_EQ(1, mock_program->num_calls());
+ }
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__no_tests);
+ATF_TEST_CASE_BODY(scanner__with_filters__no_tests)
+{
+ const model::test_programs_vector test_programs;
+
+ std::set< engine::test_filter > filters;
+ filters.insert(engine::test_filter(fs::path("foo"), "bar"));
+
+ engine::scanner scanner(test_programs, filters);
+ ATF_REQUIRE(scanner.done());
+ ATF_REQUIRE(!scanner.yield());
+ ATF_REQUIRE_EQ(filters, scanner.unused_filters());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__no_matches);
+ATF_TEST_CASE_BODY(scanner__with_filters__no_matches)
+{
+ const model::test_program_ptr test_program1 = new_test_program(
+ "dir/program1", "foo_test", "bar_test", "baz_test", NULL);
+ const model::test_program_ptr test_program2 = new_test_program(
+ "dir/program2", "bar_test", NULL);
+ const model::test_program_ptr test_program3 = new_test_program(
+ "program3", "another_test", "last_test", NULL);
+
+ model::test_programs_vector test_programs;
+ test_programs.push_back(test_program1);
+ test_programs.push_back(test_program2);
+ test_programs.push_back(test_program3);
+
+ std::set< engine::test_filter > filters;
+ filters.insert(engine::test_filter(fs::path("dir/program2"), "baz_test"));
+ filters.insert(engine::test_filter(fs::path("program4"), "another_test"));
+ filters.insert(engine::test_filter(fs::path("dir/program3"), ""));
+
+ const std::set< engine::scan_result > exp_results;
+
+ engine::scanner scanner(test_programs, filters);
+ const std::set< engine::scan_result > results = yield_all(scanner);
+ ATF_REQUIRE_EQ(exp_results, results);
+ ATF_REQUIRE_EQ(filters, scanner.unused_filters());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__some_matches);
+ATF_TEST_CASE_BODY(scanner__with_filters__some_matches)
+{
+ const model::test_program_ptr test_program1 = new_test_program(
+ "dir/program1", "foo_test", "bar_test", "baz_test", NULL);
+ const model::test_program_ptr test_program2 = new_test_program(
+ "dir/program2", "bar_test", NULL);
+ const model::test_program_ptr test_program3 = new_test_program(
+ "program3", "another_test", "last_test", NULL);
+ const model::test_program_ptr test_program4 = new_test_program(
+ "program4", "more_test", NULL);
+
+ model::test_programs_vector test_programs;
+ test_programs.push_back(test_program1);
+ test_programs.push_back(test_program2);
+ test_programs.push_back(test_program3);
+ test_programs.push_back(test_program4);
+
+ std::set< engine::test_filter > filters;
+ filters.insert(engine::test_filter(fs::path("dir/program1"), "baz_test"));
+ filters.insert(engine::test_filter(fs::path("dir/program2"), "foo_test"));
+ filters.insert(engine::test_filter(fs::path("program3"), ""));
+
+ std::set< engine::test_filter > exp_filters;
+ exp_filters.insert(engine::test_filter(fs::path("dir/program2"),
+ "foo_test"));
+
+ std::set< engine::scan_result > exp_results;
+ exp_results.insert(engine::scan_result(test_program1, "baz_test"));
+ exp_results.insert(engine::scan_result(test_program3, "another_test"));
+ exp_results.insert(engine::scan_result(test_program3, "last_test"));
+
+ engine::scanner scanner(test_programs, filters);
+ const std::set< engine::scan_result > results = yield_all(scanner);
+ ATF_REQUIRE_EQ(exp_results, results);
+
+ ATF_REQUIRE_EQ(exp_filters, scanner.unused_filters());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__verify_lazy_loads);
+ATF_TEST_CASE_BODY(scanner__with_filters__verify_lazy_loads)
+{
+ const model::test_program_ptr test_program1(new mock_test_program(
+ fs::path("first")));
+ const mock_test_program* mock_program1 =
+ dynamic_cast< const mock_test_program* >(test_program1.get());
+ const model::test_program_ptr test_program2(new mock_test_program(
+ fs::path("second")));
+ const mock_test_program* mock_program2 =
+ dynamic_cast< const mock_test_program* >(test_program2.get());
+
+ model::test_programs_vector test_programs;
+ test_programs.push_back(test_program1);
+ test_programs.push_back(test_program2);
+
+ std::set< engine::test_filter > filters;
+ filters.insert(engine::test_filter(fs::path("first"), ""));
+
+ std::set< engine::scan_result > exp_results;
+ exp_results.insert(engine::scan_result(test_program1, "one"));
+ exp_results.insert(engine::scan_result(test_program1, "two"));
+
+ engine::scanner scanner(test_programs, filters);
+ std::set< engine::scan_result > results;
+ ATF_REQUIRE_EQ(0, mock_program1->num_calls());
+ ATF_REQUIRE_EQ(0, mock_program2->num_calls());
+
+ results.insert(scanner.yield().get());
+ ATF_REQUIRE_EQ(1, mock_program1->num_calls());
+ ATF_REQUIRE_EQ(0, mock_program2->num_calls());
+ results.insert(scanner.yield().get());
+ ATF_REQUIRE_EQ(1, mock_program1->num_calls());
+ ATF_REQUIRE_EQ(0, mock_program2->num_calls());
+ ATF_REQUIRE(scanner.done());
+
+ ATF_REQUIRE_EQ(exp_results, results);
+ ATF_REQUIRE(scanner.unused_filters().empty());
+
+ ATF_REQUIRE_EQ(1, mock_program1->num_calls());
+ ATF_REQUIRE_EQ(0, mock_program2->num_calls());
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, scanner__no_filters__no_tests);
+ ATF_ADD_TEST_CASE(tcs, scanner__no_filters__one_test_in_one_program);
+ ATF_ADD_TEST_CASE(tcs, scanner__no_filters__one_test_per_many_programs);
+ ATF_ADD_TEST_CASE(tcs, scanner__no_filters__many_tests_in_one_program);
+ ATF_ADD_TEST_CASE(tcs, scanner__no_filters__many_tests_per_many_programs);
+ ATF_ADD_TEST_CASE(tcs, scanner__no_filters__verify_lazy_loads);
+
+ ATF_ADD_TEST_CASE(tcs, scanner__with_filters__no_tests);
+ ATF_ADD_TEST_CASE(tcs, scanner__with_filters__no_matches);
+ ATF_ADD_TEST_CASE(tcs, scanner__with_filters__some_matches);
+ ATF_ADD_TEST_CASE(tcs, scanner__with_filters__verify_lazy_loads);
+}
diff --git a/engine/scheduler.cpp b/engine/scheduler.cpp
new file mode 100644
index 000000000000..e7b51d23acca
--- /dev/null
+++ b/engine/scheduler.cpp
@@ -0,0 +1,1373 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/scheduler.hpp"
+
+extern "C" {
+#include <unistd.h>
+}
+
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <memory>
+#include <stdexcept>
+
+#include "engine/config.hpp"
+#include "engine/exceptions.hpp"
+#include "engine/requirements.hpp"
+#include "model/context.hpp"
+#include "model/metadata.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "model/test_result.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/datetime.hpp"
+#include "utils/defs.hpp"
+#include "utils/env.hpp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/directory.hpp"
+#include "utils/fs/exceptions.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/logging/macros.hpp"
+#include "utils/noncopyable.hpp"
+#include "utils/optional.ipp"
+#include "utils/passwd.hpp"
+#include "utils/process/executor.ipp"
+#include "utils/process/status.hpp"
+#include "utils/sanity.hpp"
+#include "utils/stacktrace.hpp"
+#include "utils/stream.hpp"
+#include "utils/text/operations.ipp"
+
+namespace config = utils::config;
+namespace datetime = utils::datetime;
+namespace executor = utils::process::executor;
+namespace fs = utils::fs;
+namespace logging = utils::logging;
+namespace passwd = utils::passwd;
+namespace process = utils::process;
+namespace scheduler = engine::scheduler;
+namespace text = utils::text;
+
+using utils::none;
+using utils::optional;
+
+
+/// Timeout for the test case cleanup operation.
+///
+/// TODO(jmmv): This is here only for testing purposes. Maybe we should expose
+/// this setting as part of the user_config.
+datetime::delta scheduler::cleanup_timeout(60, 0);
+
+
+/// Timeout for the test case listing operation.
+///
+/// TODO(jmmv): This is here only for testing purposes. Maybe we should expose
+/// this setting as part of the user_config.
+datetime::delta scheduler::list_timeout(300, 0);
+
+
+namespace {
+
+
+/// Magic exit status to indicate that the test case was probably skipped.
+///
+/// The test case was only skipped if and only if we return this exit code and
+/// we find the skipped_cookie file on disk.
+static const int exit_skipped = 84;
+
+
+/// Text file containing the skip reason for the test case.
+///
+/// This will only be present within unique_work_directory if the test case
+/// exited with the exit_skipped code. However, there is no guarantee that the
+/// file is there (say if the test really decided to exit with code exit_skipped
+/// on its own).
+static const char* skipped_cookie = "skipped.txt";
+
+
+/// Mapping of interface names to interface definitions.
+typedef std::map< std::string, std::shared_ptr< scheduler::interface > >
+ interfaces_map;
+
+
+/// Mapping of interface names to interface definitions.
+///
+/// Use register_interface() to add an entry to this global table.
+static interfaces_map interfaces;
+
+
+/// Scans the contents of a directory and appends the file listing to a file.
+///
+/// \param dir_path The directory to scan.
+/// \param output_file The file to which to append the listing.
+///
+/// \throw engine::error If there are problems listing the files.
+static void
+append_files_listing(const fs::path& dir_path, const fs::path& output_file)
+{
+ std::ofstream output(output_file.c_str(), std::ios::app);
+ if (!output)
+ throw engine::error(F("Failed to open output file %s for append")
+ % output_file);
+ try {
+ std::set < std::string > names;
+
+ const fs::directory dir(dir_path);
+ for (fs::directory::const_iterator iter = dir.begin();
+ iter != dir.end(); ++iter) {
+ if (iter->name != "." && iter->name != "..")
+ names.insert(iter->name);
+ }
+
+ if (!names.empty()) {
+ output << "Files left in work directory after failure: "
+ << text::join(names, ", ") << '\n';
+ }
+ } catch (const fs::error& e) {
+ throw engine::error(F("Cannot append files listing to %s: %s")
+ % output_file % e.what());
+ }
+}
+
+
+/// Maintenance data held while a test is being executed.
+///
+/// This data structure exists from the moment when a test is executed via
+/// scheduler::spawn_test() or scheduler::impl::spawn_cleanup() to when it is
+/// cleaned up with result_handle::cleanup().
+///
+/// This is a base data type intended to be extended for the test and cleanup
+/// cases so that each contains only the relevant data.
+struct exec_data : utils::noncopyable {
+ /// Test program data for this test case.
+ const model::test_program_ptr test_program;
+
+ /// Name of the test case.
+ const std::string test_case_name;
+
+ /// Constructor.
+ ///
+ /// \param test_program_ Test program data for this test case.
+ /// \param test_case_name_ Name of the test case.
+ exec_data(const model::test_program_ptr test_program_,
+ const std::string& test_case_name_) :
+ test_program(test_program_), test_case_name(test_case_name_)
+ {
+ }
+
+ /// Destructor.
+ virtual ~exec_data(void)
+ {
+ }
+};
+
+
+/// Maintenance data held while a test is being executed.
+struct test_exec_data : public exec_data {
+ /// Test program-specific execution interface.
+ const std::shared_ptr< scheduler::interface > interface;
+
+ /// User configuration passed to the execution of the test. We need this
+ /// here to recover it later when chaining the execution of a cleanup
+ /// routine (if any).
+ const config::tree user_config;
+
+ /// Whether this test case still needs to have its cleanup routine executed.
+ ///
+ /// This is set externally when the cleanup routine is actually invoked to
+ /// denote that no further attempts shall be made at cleaning this up.
+ bool needs_cleanup;
+
+ /// The exit_handle for this test once it has completed.
+ ///
+ /// This is set externally when the test case has finished, as we need this
+ /// information to invoke the followup cleanup routine in the right context,
+ /// as indicated by needs_cleanup.
+ optional< executor::exit_handle > exit_handle;
+
+ /// Constructor.
+ ///
+ /// \param test_program_ Test program data for this test case.
+ /// \param test_case_name_ Name of the test case.
+ /// \param interface_ Test program-specific execution interface.
+ /// \param user_config_ User configuration passed to the test.
+ test_exec_data(const model::test_program_ptr test_program_,
+ const std::string& test_case_name_,
+ const std::shared_ptr< scheduler::interface > interface_,
+ const config::tree& user_config_) :
+ exec_data(test_program_, test_case_name_),
+ interface(interface_), user_config(user_config_)
+ {
+ const model::test_case& test_case = test_program->find(test_case_name);
+ needs_cleanup = test_case.get_metadata().has_cleanup();
+ }
+};
+
+
+/// Maintenance data held while a test cleanup routine is being executed.
+///
+/// Instances of this object are related to a previous test_exec_data, as
+/// cleanup routines can only exist once the test has been run.
+struct cleanup_exec_data : public exec_data {
+ /// The exit handle of the test. This is necessary so that we can return
+ /// the correct exit_handle to the user of the scheduler.
+ executor::exit_handle body_exit_handle;
+
+ /// The final result of the test's body. This is necessary to compute the
+ /// right return value for a test with a cleanup routine: the body result is
+ /// respected if it is a "bad" result; else the result of the cleanup
+ /// routine is used if it has failed.
+ model::test_result body_result;
+
+ /// Constructor.
+ ///
+ /// \param test_program_ Test program data for this test case.
+ /// \param test_case_name_ Name of the test case.
+ /// \param body_exit_handle_ If not none, exit handle of the body
+ /// corresponding to the cleanup routine represented by this exec_data.
+ /// \param body_result_ If not none, result of the body corresponding to the
+ /// cleanup routine represented by this exec_data.
+ cleanup_exec_data(const model::test_program_ptr test_program_,
+ const std::string& test_case_name_,
+ const executor::exit_handle& body_exit_handle_,
+ const model::test_result& body_result_) :
+ exec_data(test_program_, test_case_name_),
+ body_exit_handle(body_exit_handle_), body_result(body_result_)
+ {
+ }
+};
+
+
+/// Shared pointer to exec_data.
+///
+/// We require this because we want exec_data to not be copyable, and thus we
+/// cannot just store it in the map without move constructors.
+typedef std::shared_ptr< exec_data > exec_data_ptr;
+
+
+/// Mapping of active PIDs to their maintenance data.
+typedef std::map< int, exec_data_ptr > exec_data_map;
+
+
+/// Enforces a test program to hold an absolute path.
+///
+/// TODO(jmmv): This function (which is a pretty ugly hack) exists because we
+/// want the interface hooks to receive a test_program as their argument.
+/// However, those hooks run after the test program has been isolated, which
+/// means that the current directory has changed since when the test_program
+/// objects were created. This causes the absolute_path() method of
+/// test_program to return bogus values if the internal representation of their
+/// path is relative. We should fix somehow: maybe making the fs module grab
+/// its "current_path" view at program startup time; or maybe by grabbing the
+/// current path at test_program creation time; or maybe something else.
+///
+/// \param program The test program to modify.
+///
+/// \return A new test program whose internal paths are absolute.
+static model::test_program
+force_absolute_paths(const model::test_program program)
+{
+ const std::string& relative = program.relative_path().str();
+ const std::string absolute = program.absolute_path().str();
+
+ const std::string root = absolute.substr(
+ 0, absolute.length() - relative.length());
+
+ return model::test_program(
+ program.interface_name(),
+ program.relative_path(), fs::path(root),
+ program.test_suite_name(),
+ program.get_metadata(), program.test_cases());
+}
+
+
+/// Functor to list the test cases of a test program.
+class list_test_cases {
+ /// Interface of the test program to execute.
+ std::shared_ptr< scheduler::interface > _interface;
+
+ /// Test program to execute.
+ const model::test_program _test_program;
+
+ /// User-provided configuration variables.
+ const config::tree& _user_config;
+
+public:
+ /// Constructor.
+ ///
+ /// \param interface Interface of the test program to execute.
+ /// \param test_program Test program to execute.
+ /// \param user_config User-provided configuration variables.
+ list_test_cases(
+ const std::shared_ptr< scheduler::interface > interface,
+ const model::test_program* test_program,
+ const config::tree& user_config) :
+ _interface(interface),
+ _test_program(force_absolute_paths(*test_program)),
+ _user_config(user_config)
+ {
+ }
+
+ /// Body of the subprocess.
+ void
+ operator()(const fs::path& /* control_directory */)
+ {
+ const config::properties_map vars = scheduler::generate_config(
+ _user_config, _test_program.test_suite_name());
+ _interface->exec_list(_test_program, vars);
+ }
+};
+
+
+/// Functor to execute a test program in a child process.
+class run_test_program {
+ /// Interface of the test program to execute.
+ std::shared_ptr< scheduler::interface > _interface;
+
+ /// Test program to execute.
+ const model::test_program _test_program;
+
+ /// Name of the test case to execute.
+ const std::string& _test_case_name;
+
+ /// User-provided configuration variables.
+ const config::tree& _user_config;
+
+ /// Verifies if the test case needs to be skipped or not.
+ ///
+ /// We could very well run this on the scheduler parent process before
+ /// issuing the fork. However, doing this here in the child process is
+ /// better for two reasons: first, it allows us to continue using the simple
+ /// spawn/wait abstraction of the scheduler; and, second, we parallelize the
+ /// requirements checks among tests.
+ ///
+ /// \post If the test's preconditions are not met, the caller process is
+ /// terminated with a special exit code and a "skipped cookie" is written to
+ /// the disk with the reason for the failure.
+ ///
+ /// \param skipped_cookie_path File to create with the skip reason details
+ /// if this test is skipped.
+ void
+ do_requirements_check(const fs::path& skipped_cookie_path)
+ {
+ const model::test_case& test_case = _test_program.find(
+ _test_case_name);
+
+ const std::string skip_reason = engine::check_reqs(
+ test_case.get_metadata(), _user_config,
+ _test_program.test_suite_name(),
+ fs::current_path());
+ if (skip_reason.empty())
+ return;
+
+ std::ofstream output(skipped_cookie_path.c_str());
+ if (!output) {
+ std::perror((F("Failed to open %s for write") %
+ skipped_cookie_path).str().c_str());
+ std::abort();
+ }
+ output << skip_reason;
+ output.close();
+
+ // Abruptly terminate the process. We don't want to run any destructors
+ // inherited from the parent process by mistake, which could, for
+ // example, delete our own control files!
+ ::_exit(exit_skipped);
+ }
+
+public:
+ /// Constructor.
+ ///
+ /// \param interface Interface of the test program to execute.
+ /// \param test_program Test program to execute.
+ /// \param test_case_name Name of the test case to execute.
+ /// \param user_config User-provided configuration variables.
+ run_test_program(
+ const std::shared_ptr< scheduler::interface > interface,
+ const model::test_program_ptr test_program,
+ const std::string& test_case_name,
+ const config::tree& user_config) :
+ _interface(interface),
+ _test_program(force_absolute_paths(*test_program)),
+ _test_case_name(test_case_name),
+ _user_config(user_config)
+ {
+ }
+
+ /// Body of the subprocess.
+ ///
+ /// \param control_directory The testcase directory where files will be
+ /// read from.
+ void
+ operator()(const fs::path& control_directory)
+ {
+ const model::test_case& test_case = _test_program.find(
+ _test_case_name);
+ if (test_case.fake_result())
+ ::_exit(EXIT_SUCCESS);
+
+ do_requirements_check(control_directory / skipped_cookie);
+
+ const config::properties_map vars = scheduler::generate_config(
+ _user_config, _test_program.test_suite_name());
+ _interface->exec_test(_test_program, _test_case_name, vars,
+ control_directory);
+ }
+};
+
+
+/// Functor to execute a test program in a child process.
+class run_test_cleanup {
+ /// Interface of the test program to execute.
+ std::shared_ptr< scheduler::interface > _interface;
+
+ /// Test program to execute.
+ const model::test_program _test_program;
+
+ /// Name of the test case to execute.
+ const std::string& _test_case_name;
+
+ /// User-provided configuration variables.
+ const config::tree& _user_config;
+
+public:
+ /// Constructor.
+ ///
+ /// \param interface Interface of the test program to execute.
+ /// \param test_program Test program to execute.
+ /// \param test_case_name Name of the test case to execute.
+ /// \param user_config User-provided configuration variables.
+ run_test_cleanup(
+ const std::shared_ptr< scheduler::interface > interface,
+ const model::test_program_ptr test_program,
+ const std::string& test_case_name,
+ const config::tree& user_config) :
+ _interface(interface),
+ _test_program(force_absolute_paths(*test_program)),
+ _test_case_name(test_case_name),
+ _user_config(user_config)
+ {
+ }
+
+ /// Body of the subprocess.
+ ///
+ /// \param control_directory The testcase directory where cleanup will be
+ /// run from.
+ void
+ operator()(const fs::path& control_directory)
+ {
+ const config::properties_map vars = scheduler::generate_config(
+ _user_config, _test_program.test_suite_name());
+ _interface->exec_cleanup(_test_program, _test_case_name, vars,
+ control_directory);
+ }
+};
+
+
+/// Obtains the right scheduler interface for a given test program.
+///
+/// \param name The name of the interface of the test program.
+///
+/// \return An scheduler interface.
+std::shared_ptr< scheduler::interface >
+find_interface(const std::string& name)
+{
+ const interfaces_map::const_iterator iter = interfaces.find(name);
+ PRE(interfaces.find(name) != interfaces.end());
+ return (*iter).second;
+}
+
+
+} // anonymous namespace
+
+
+void
+scheduler::interface::exec_cleanup(
+ const model::test_program& /* test_program */,
+ const std::string& /* test_case_name */,
+ const config::properties_map& /* vars */,
+ const utils::fs::path& /* control_directory */) const
+{
+ // Most test interfaces do not support standalone cleanup routines so
+ // provide a default implementation that does nothing.
+ UNREACHABLE_MSG("exec_cleanup not implemented for an interface that "
+ "supports standalone cleanup routines");
+}
+
+
+/// Internal implementation of a lazy_test_program.
+struct engine::scheduler::lazy_test_program::impl : utils::noncopyable {
+ /// Whether the test cases list has been yet loaded or not.
+ bool _loaded;
+
+ /// User configuration to pass to the test program list operation.
+ config::tree _user_config;
+
+ /// Scheduler context to use to load test cases.
+ scheduler::scheduler_handle& _scheduler_handle;
+
+ /// Constructor.
+ ///
+ /// \param user_config_ User configuration to pass to the test program list
+ /// operation.
+ /// \param scheduler_handle_ Scheduler context to use when loading test
+ /// cases.
+ impl(const config::tree& user_config_,
+ scheduler::scheduler_handle& scheduler_handle_) :
+ _loaded(false), _user_config(user_config_),
+ _scheduler_handle(scheduler_handle_)
+ {
+ }
+};
+
+
+/// Constructs a new test program.
+///
+/// \param interface_name_ Name of the test program interface.
+/// \param binary_ The name of the test program binary relative to root_.
+/// \param root_ The root of the test suite containing the test program.
+/// \param test_suite_name_ The name of the test suite this program belongs to.
+/// \param md_ Metadata of the test program.
+/// \param user_config_ User configuration to pass to the scheduler.
+/// \param scheduler_handle_ Scheduler context to use to load test cases.
+scheduler::lazy_test_program::lazy_test_program(
+ const std::string& interface_name_,
+ const fs::path& binary_,
+ const fs::path& root_,
+ const std::string& test_suite_name_,
+ const model::metadata& md_,
+ const config::tree& user_config_,
+ scheduler::scheduler_handle& scheduler_handle_) :
+ test_program(interface_name_, binary_, root_, test_suite_name_, md_,
+ model::test_cases_map()),
+ _pimpl(new impl(user_config_, scheduler_handle_))
+{
+}
+
+
+/// Gets or loads the list of test cases from the test program.
+///
+/// \return The list of test cases provided by the test program.
+const model::test_cases_map&
+scheduler::lazy_test_program::test_cases(void) const
+{
+ _pimpl->_scheduler_handle.check_interrupt();
+
+ if (!_pimpl->_loaded) {
+ const model::test_cases_map tcs = _pimpl->_scheduler_handle.list_tests(
+ this, _pimpl->_user_config);
+
+ // Due to the restrictions on when set_test_cases() may be called (as a
+ // way to lazily initialize the test cases list before it is ever
+ // returned), this cast is valid.
+ const_cast< scheduler::lazy_test_program* >(this)->set_test_cases(tcs);
+
+ _pimpl->_loaded = true;
+
+ _pimpl->_scheduler_handle.check_interrupt();
+ }
+
+ INV(_pimpl->_loaded);
+ return test_program::test_cases();
+}
+
+
+/// Internal implementation for the result_handle class.
+struct engine::scheduler::result_handle::bimpl : utils::noncopyable {
+ /// Generic executor exit handle for this result handle.
+ executor::exit_handle generic;
+
+ /// Mutable pointer to the corresponding scheduler state.
+ ///
+ /// This object references a member of the scheduler_handle that yielded
+ /// this result_handle instance. We need this direct access to clean up
+ /// after ourselves when the result is destroyed.
+ exec_data_map& all_exec_data;
+
+ /// Constructor.
+ ///
+ /// \param generic_ Generic executor exit handle for this result handle.
+ /// \param [in,out] all_exec_data_ Global object keeping track of all active
+ /// executions for an scheduler. This is a pointer to a member of the
+ /// scheduler_handle object.
+ bimpl(const executor::exit_handle generic_, exec_data_map& all_exec_data_) :
+ generic(generic_), all_exec_data(all_exec_data_)
+ {
+ }
+
+ /// Destructor.
+ ~bimpl(void)
+ {
+ LD(F("Removing %s from all_exec_data") % generic.original_pid());
+ all_exec_data.erase(generic.original_pid());
+ }
+};
+
+
+/// Constructor.
+///
+/// \param pbimpl Constructed internal implementation.
+scheduler::result_handle::result_handle(std::shared_ptr< bimpl > pbimpl) :
+ _pbimpl(pbimpl)
+{
+}
+
+
+/// Destructor.
+scheduler::result_handle::~result_handle(void)
+{
+}
+
+
+/// Cleans up the test case results.
+///
+/// This function should be called explicitly as it provides the means to
+/// control any exceptions raised during cleanup. Do not rely on the destructor
+/// to clean things up.
+///
+/// \throw engine::error If the cleanup fails, especially due to the inability
+/// to remove the work directory.
+void
+scheduler::result_handle::cleanup(void)
+{
+ _pbimpl->generic.cleanup();
+}
+
+
+/// Returns the original PID corresponding to this result.
+///
+/// \return An exec_handle.
+int
+scheduler::result_handle::original_pid(void) const
+{
+ return _pbimpl->generic.original_pid();
+}
+
+
+/// Returns the timestamp of when spawn_test was called.
+///
+/// \return A timestamp.
+const datetime::timestamp&
+scheduler::result_handle::start_time(void) const
+{
+ return _pbimpl->generic.start_time();
+}
+
+
+/// Returns the timestamp of when wait_any_test returned this object.
+///
+/// \return A timestamp.
+const datetime::timestamp&
+scheduler::result_handle::end_time(void) const
+{
+ return _pbimpl->generic.end_time();
+}
+
+
+/// Returns the path to the test-specific work directory.
+///
+/// This is guaranteed to be clear of files created by the scheduler.
+///
+/// \return The path to a directory that exists until cleanup() is called.
+fs::path
+scheduler::result_handle::work_directory(void) const
+{
+ return _pbimpl->generic.work_directory();
+}
+
+
+/// Returns the path to the test's stdout file.
+///
+/// \return The path to a file that exists until cleanup() is called.
+const fs::path&
+scheduler::result_handle::stdout_file(void) const
+{
+ return _pbimpl->generic.stdout_file();
+}
+
+
+/// Returns the path to the test's stderr file.
+///
+/// \return The path to a file that exists until cleanup() is called.
+const fs::path&
+scheduler::result_handle::stderr_file(void) const
+{
+ return _pbimpl->generic.stderr_file();
+}
+
+
+/// Internal implementation for the test_result_handle class.
+struct engine::scheduler::test_result_handle::impl : utils::noncopyable {
+ /// Test program data for this test case.
+ model::test_program_ptr test_program;
+
+ /// Name of the test case.
+ std::string test_case_name;
+
+ /// The actual result of the test execution.
+ const model::test_result test_result;
+
+ /// Constructor.
+ ///
+ /// \param test_program_ Test program data for this test case.
+ /// \param test_case_name_ Name of the test case.
+ /// \param test_result_ The actual result of the test execution.
+ impl(const model::test_program_ptr test_program_,
+ const std::string& test_case_name_,
+ const model::test_result& test_result_) :
+ test_program(test_program_),
+ test_case_name(test_case_name_),
+ test_result(test_result_)
+ {
+ }
+};
+
+
+/// Constructor.
+///
+/// \param pbimpl Constructed internal implementation for the base object.
+/// \param pimpl Constructed internal implementation.
+scheduler::test_result_handle::test_result_handle(
+ std::shared_ptr< bimpl > pbimpl, std::shared_ptr< impl > pimpl) :
+ result_handle(pbimpl), _pimpl(pimpl)
+{
+}
+
+
+/// Destructor.
+scheduler::test_result_handle::~test_result_handle(void)
+{
+}
+
+
+/// Returns the test program that yielded this result.
+///
+/// \return A test program.
+const model::test_program_ptr
+scheduler::test_result_handle::test_program(void) const
+{
+ return _pimpl->test_program;
+}
+
+
+/// Returns the name of the test case that yielded this result.
+///
+/// \return A test case name
+const std::string&
+scheduler::test_result_handle::test_case_name(void) const
+{
+ return _pimpl->test_case_name;
+}
+
+
+/// Returns the actual result of the test execution.
+///
+/// \return A test result.
+const model::test_result&
+scheduler::test_result_handle::test_result(void) const
+{
+ return _pimpl->test_result;
+}
+
+
+/// Internal implementation for the scheduler_handle.
+struct engine::scheduler::scheduler_handle::impl : utils::noncopyable {
+ /// Generic executor instance encapsulated by this one.
+ executor::executor_handle generic;
+
+ /// Mapping of exec handles to the data required at run time.
+ exec_data_map all_exec_data;
+
+ /// Collection of test_exec_data objects.
+ typedef std::vector< const test_exec_data* > test_exec_data_vector;
+
+ /// Constructor.
+ impl(void) : generic(executor::setup())
+ {
+ }
+
+ /// Destructor.
+ ///
+ /// This runs any pending cleanup routines, which should only happen if the
+ /// scheduler is abruptly terminated (aka if a signal is received).
+ ~impl(void)
+ {
+ const test_exec_data_vector tests_data = tests_needing_cleanup();
+
+ for (test_exec_data_vector::const_iterator iter = tests_data.begin();
+ iter != tests_data.end(); ++iter) {
+ const test_exec_data* test_data = *iter;
+
+ try {
+ sync_cleanup(test_data);
+ } catch (const std::runtime_error& e) {
+ LW(F("Failed to run cleanup routine for %s:%s on abrupt "
+ "termination")
+ % test_data->test_program->relative_path()
+ % test_data->test_case_name);
+ }
+ }
+ }
+
+ /// Finds any pending exec_datas that correspond to tests needing cleanup.
+ ///
+ /// \return The collection of test_exec_data objects that have their
+ /// needs_cleanup property set to true.
+ test_exec_data_vector
+ tests_needing_cleanup(void)
+ {
+ test_exec_data_vector tests_data;
+
+ for (exec_data_map::const_iterator iter = all_exec_data.begin();
+ iter != all_exec_data.end(); ++iter) {
+ const exec_data_ptr data = (*iter).second;
+
+ try {
+ test_exec_data* test_data = &dynamic_cast< test_exec_data& >(
+ *data.get());
+ if (test_data->needs_cleanup) {
+ tests_data.push_back(test_data);
+ test_data->needs_cleanup = false;
+ }
+ } catch (const std::bad_cast& e) {
+ // Do nothing for cleanup_exec_data objects.
+ }
+ }
+
+ return tests_data;
+ }
+
+ /// Cleans up a single test case synchronously.
+ ///
+ /// \param test_data The data of the previously executed test case to be
+ /// cleaned up.
+ void
+ sync_cleanup(const test_exec_data* test_data)
+ {
+ // The message in this result should never be seen by the user, but use
+ // something reasonable just in case it leaks and we need to pinpoint
+ // the call site.
+ model::test_result result(model::test_result_broken,
+ "Test case died abruptly");
+
+ const executor::exec_handle cleanup_handle = spawn_cleanup(
+ test_data->test_program, test_data->test_case_name,
+ test_data->user_config, test_data->exit_handle.get(),
+ result);
+ generic.wait(cleanup_handle);
+ }
+
+ /// Forks and executes a test case cleanup routine asynchronously.
+ ///
+ /// \param test_program The container test program.
+ /// \param test_case_name The name of the test case to run.
+ /// \param user_config User-provided configuration variables.
+ /// \param body_handle The exit handle of the test case's corresponding
+ /// body. The cleanup will be executed in the same context.
+ /// \param body_result The result of the test case's corresponding body.
+ ///
+ /// \return A handle for the background operation. Used to match the result
+ /// of the execution returned by wait_any() with this invocation.
+ executor::exec_handle
+ spawn_cleanup(const model::test_program_ptr test_program,
+ const std::string& test_case_name,
+ const config::tree& user_config,
+ const executor::exit_handle& body_handle,
+ const model::test_result& body_result)
+ {
+ generic.check_interrupt();
+
+ const std::shared_ptr< scheduler::interface > interface =
+ find_interface(test_program->interface_name());
+
+ LI(F("Spawning %s:%s (cleanup)") % test_program->absolute_path() %
+ test_case_name);
+
+ const executor::exec_handle handle = generic.spawn_followup(
+ run_test_cleanup(interface, test_program, test_case_name,
+ user_config),
+ body_handle, cleanup_timeout);
+
+ const exec_data_ptr data(new cleanup_exec_data(
+ test_program, test_case_name, body_handle, body_result));
+ LD(F("Inserting %s into all_exec_data (cleanup)") % handle.pid());
+ INV_MSG(all_exec_data.find(handle.pid()) == all_exec_data.end(),
+ F("PID %s already in all_exec_data; not properly cleaned "
+ "up or reused too fast") % handle.pid());;
+ all_exec_data.insert(exec_data_map::value_type(handle.pid(), data));
+
+ return handle;
+ }
+};
+
+
+/// Constructor.
+scheduler::scheduler_handle::scheduler_handle(void) : _pimpl(new impl())
+{
+}
+
+
+/// Destructor.
+scheduler::scheduler_handle::~scheduler_handle(void)
+{
+}
+
+
+/// Queries the path to the root of the work directory for all tests.
+///
+/// \return A path.
+const fs::path&
+scheduler::scheduler_handle::root_work_directory(void) const
+{
+ return _pimpl->generic.root_work_directory();
+}
+
+
+/// Cleans up the scheduler state.
+///
+/// This function should be called explicitly as it provides the means to
+/// control any exceptions raised during cleanup. Do not rely on the destructor
+/// to clean things up.
+///
+/// \throw engine::error If there are problems cleaning up the scheduler.
+void
+scheduler::scheduler_handle::cleanup(void)
+{
+ _pimpl->generic.cleanup();
+}
+
+
+/// Checks if the given interface name is valid.
+///
+/// \param name The name of the interface to validate.
+///
+/// \throw engine::error If the given interface is not supported.
+void
+scheduler::ensure_valid_interface(const std::string& name)
+{
+ if (interfaces.find(name) == interfaces.end())
+ throw engine::error(F("Unsupported test interface '%s'") % name);
+}
+
+
+/// Registers a new interface.
+///
+/// \param name The name of the interface. Must not have yet been registered.
+/// \param spec Interface specification.
+void
+scheduler::register_interface(const std::string& name,
+ const std::shared_ptr< interface > spec)
+{
+ PRE(interfaces.find(name) == interfaces.end());
+ interfaces.insert(interfaces_map::value_type(name, spec));
+}
+
+
+/// Returns the names of all registered interfaces.
+///
+/// \return A collection of interface names.
+std::set< std::string >
+scheduler::registered_interface_names(void)
+{
+ std::set< std::string > names;
+ for (interfaces_map::const_iterator iter = interfaces.begin();
+ iter != interfaces.end(); ++iter) {
+ names.insert((*iter).first);
+ }
+ return names;
+}
+
+
+/// Initializes the scheduler.
+///
+/// \pre This function can only be called if there is no other scheduler_handle
+/// object alive.
+///
+/// \return A handle to the operations of the scheduler.
+scheduler::scheduler_handle
+scheduler::setup(void)
+{
+ return scheduler_handle();
+}
+
+
+/// Retrieves the list of test cases from a test program.
+///
+/// This operation is currently synchronous.
+///
+/// This operation should never throw. Any errors during the processing of the
+/// test case list are subsumed into a single test case in the return value that
+/// represents the failed retrieval.
+///
+/// \param test_program The test program from which to obtain the list of test
+/// cases.
+/// \param user_config User-provided configuration variables.
+///
+/// \return The list of test cases.
+model::test_cases_map
+scheduler::scheduler_handle::list_tests(
+ const model::test_program* test_program,
+ const config::tree& user_config)
+{
+ _pimpl->generic.check_interrupt();
+
+ const std::shared_ptr< scheduler::interface > interface = find_interface(
+ test_program->interface_name());
+
+ try {
+ const executor::exec_handle exec_handle = _pimpl->generic.spawn(
+ list_test_cases(interface, test_program, user_config),
+ list_timeout, none);
+ executor::exit_handle exit_handle = _pimpl->generic.wait(exec_handle);
+
+ const model::test_cases_map test_cases = interface->parse_list(
+ exit_handle.status(),
+ exit_handle.stdout_file(),
+ exit_handle.stderr_file());
+
+ exit_handle.cleanup();
+
+ if (test_cases.empty())
+ throw std::runtime_error("Empty test cases list");
+
+ return test_cases;
+ } catch (const std::runtime_error& e) {
+ // TODO(jmmv): This is a very ugly workaround for the fact that we
+ // cannot report failures at the test-program level.
+ LW(F("Failed to load test cases list: %s") % e.what());
+ model::test_cases_map fake_test_cases;
+ fake_test_cases.insert(model::test_cases_map::value_type(
+ "__test_cases_list__",
+ model::test_case(
+ "__test_cases_list__",
+ "Represents the correct processing of the test cases list",
+ model::test_result(model::test_result_broken, e.what()))));
+ return fake_test_cases;
+ }
+}
+
+
+/// Forks and executes a test case asynchronously.
+///
+/// Note that the caller needn't know if the test has a cleanup routine or not.
+/// If there indeed is a cleanup routine, we trigger it at wait_any() time.
+///
+/// \param test_program The container test program.
+/// \param test_case_name The name of the test case to run.
+/// \param user_config User-provided configuration variables.
+///
+/// \return A handle for the background operation. Used to match the result of
+/// the execution returned by wait_any() with this invocation.
+scheduler::exec_handle
+scheduler::scheduler_handle::spawn_test(
+ const model::test_program_ptr test_program,
+ const std::string& test_case_name,
+ const config::tree& user_config)
+{
+ _pimpl->generic.check_interrupt();
+
+ const std::shared_ptr< scheduler::interface > interface = find_interface(
+ test_program->interface_name());
+
+ LI(F("Spawning %s:%s") % test_program->absolute_path() % test_case_name);
+
+ const model::test_case& test_case = test_program->find(test_case_name);
+
+ optional< passwd::user > unprivileged_user;
+ if (user_config.is_set("unprivileged_user") &&
+ test_case.get_metadata().required_user() == "unprivileged") {
+ unprivileged_user = user_config.lookup< engine::user_node >(
+ "unprivileged_user");
+ }
+
+ const executor::exec_handle handle = _pimpl->generic.spawn(
+ run_test_program(interface, test_program, test_case_name,
+ user_config),
+ test_case.get_metadata().timeout(),
+ unprivileged_user);
+
+ const exec_data_ptr data(new test_exec_data(
+ test_program, test_case_name, interface, user_config));
+ LD(F("Inserting %s into all_exec_data") % handle.pid());
+ INV_MSG(
+ _pimpl->all_exec_data.find(handle.pid()) == _pimpl->all_exec_data.end(),
+ F("PID %s already in all_exec_data; not cleaned up or reused too fast")
+ % handle.pid());;
+ _pimpl->all_exec_data.insert(exec_data_map::value_type(handle.pid(), data));
+
+ return handle.pid();
+}
+
+
+/// Waits for completion of any forked test case.
+///
+/// Note that if the terminated test case has a cleanup routine, this function
+/// is the one in charge of spawning the cleanup routine asynchronously.
+///
+/// \return The result of the execution of a subprocess. This is a dynamically
+/// allocated object because the scheduler can spawn subprocesses of various
+/// types and, at wait time, we don't know upfront what we are going to get.
+scheduler::result_handle_ptr
+scheduler::scheduler_handle::wait_any(void)
+{
+ _pimpl->generic.check_interrupt();
+
+ executor::exit_handle handle = _pimpl->generic.wait_any();
+
+ const exec_data_map::iterator iter = _pimpl->all_exec_data.find(
+ handle.original_pid());
+ exec_data_ptr data = (*iter).second;
+
+ utils::dump_stacktrace_if_available(data->test_program->absolute_path(),
+ _pimpl->generic, handle);
+
+ optional< model::test_result > result;
+ try {
+ test_exec_data* test_data = &dynamic_cast< test_exec_data& >(
+ *data.get());
+ LD(F("Got %s from all_exec_data") % handle.original_pid());
+
+ test_data->exit_handle = handle;
+
+ const model::test_case& test_case = test_data->test_program->find(
+ test_data->test_case_name);
+
+ result = test_case.fake_result();
+
+ if (!result && handle.status() && handle.status().get().exited() &&
+ handle.status().get().exitstatus() == exit_skipped) {
+ // If the test's process terminated with our magic "exit_skipped"
+ // status, there are two cases to handle. The first is the case
+ // where the "skipped cookie" exists, in which case we never got to
+ // actually invoke the test program; if that's the case, handle it
+ // here. The second case is where the test case actually decided to
+ // exit with the "exit_skipped" status; in that case, just fall back
+ // to the regular status handling.
+ const fs::path skipped_cookie_path = handle.control_directory() /
+ skipped_cookie;
+ std::ifstream input(skipped_cookie_path.c_str());
+ if (input) {
+ result = model::test_result(model::test_result_skipped,
+ utils::read_stream(input));
+ input.close();
+
+ // If we determined that the test needs to be skipped, we do not
+ // want to run the cleanup routine because doing so could result
+ // in errors. However, we still want to run the cleanup routine
+ // if the test's body reports a skip (because actions could have
+ // already been taken).
+ test_data->needs_cleanup = false;
+ }
+ }
+ if (!result) {
+ result = test_data->interface->compute_result(
+ handle.status(),
+ handle.control_directory(),
+ handle.stdout_file(),
+ handle.stderr_file());
+ }
+ INV(result);
+
+ if (!result.get().good()) {
+ append_files_listing(handle.work_directory(),
+ handle.stderr_file());
+ }
+
+ if (test_data->needs_cleanup) {
+ INV(test_case.get_metadata().has_cleanup());
+ // The test body has completed and we have processed it. If there
+ // is a cleanup routine, trigger it now and wait for any other test
+ // completion. The caller never knows about cleanup routines.
+ _pimpl->spawn_cleanup(test_data->test_program,
+ test_data->test_case_name,
+ test_data->user_config, handle, result.get());
+ test_data->needs_cleanup = false;
+
+ // TODO(jmmv): Chaining this call is ugly. We'd be better off by
+ // looping over terminated processes until we got a result suitable
+ // for user consumption. For the time being this is good enough and
+ // not a problem because the call chain won't get big: the majority
+ // of test cases do not have cleanup routines.
+ return wait_any();
+ }
+ } catch (const std::bad_cast& e) {
+ const cleanup_exec_data* cleanup_data =
+ &dynamic_cast< const cleanup_exec_data& >(*data.get());
+ LD(F("Got %s from all_exec_data (cleanup)") % handle.original_pid());
+
+ // Handle the completion of cleanup subprocesses internally: the caller
+ // is not aware that these exist so, when we return, we must return the
+ // data for the original test that triggered this routine. For example,
+ // because the caller wants to see the exact same exec_handle that was
+ // returned by spawn_test.
+
+ const model::test_result& body_result = cleanup_data->body_result;
+ if (body_result.good()) {
+ if (!handle.status()) {
+ result = model::test_result(model::test_result_broken,
+ "Test case cleanup timed out");
+ } else {
+ if (!handle.status().get().exited() ||
+ handle.status().get().exitstatus() != EXIT_SUCCESS) {
+ result = model::test_result(
+ model::test_result_broken,
+ "Test case cleanup did not terminate successfully");
+ } else {
+ result = body_result;
+ }
+ }
+ } else {
+ result = body_result;
+ }
+
+ // Untrack the cleanup process. This must be done explicitly because we
+ // do not create a result_handle object for the cleanup, and that is the
+ // one in charge of doing so in the regular (non-cleanup) case.
+ LD(F("Removing %s from all_exec_data (cleanup) in favor of %s")
+ % handle.original_pid()
+ % cleanup_data->body_exit_handle.original_pid());
+ _pimpl->all_exec_data.erase(handle.original_pid());
+
+ handle = cleanup_data->body_exit_handle;
+ }
+ INV(result);
+
+ std::shared_ptr< result_handle::bimpl > result_handle_bimpl(
+ new result_handle::bimpl(handle, _pimpl->all_exec_data));
+ std::shared_ptr< test_result_handle::impl > test_result_handle_impl(
+ new test_result_handle::impl(
+ data->test_program, data->test_case_name, result.get()));
+ return result_handle_ptr(new test_result_handle(result_handle_bimpl,
+ test_result_handle_impl));
+}
+
+
+/// Forks and executes a test case synchronously for debugging.
+///
+/// \pre No other processes should be in execution by the scheduler.
+///
+/// \param test_program The container test program.
+/// \param test_case_name The name of the test case to run.
+/// \param user_config User-provided configuration variables.
+/// \param stdout_target File to which to write the stdout of the test case.
+/// \param stderr_target File to which to write the stderr of the test case.
+///
+/// \return The result of the execution of the test.
+scheduler::result_handle_ptr
+scheduler::scheduler_handle::debug_test(
+ const model::test_program_ptr test_program,
+ const std::string& test_case_name,
+ const config::tree& user_config,
+ const fs::path& stdout_target,
+ const fs::path& stderr_target)
+{
+ const exec_handle exec_handle = spawn_test(
+ test_program, test_case_name, user_config);
+ result_handle_ptr result_handle = wait_any();
+
+ // TODO(jmmv): We need to do this while the subprocess is alive. This is
+ // important for debugging purposes, as we should see the contents of stdout
+ // or stderr as they come in.
+ //
+ // Unfortunately, we cannot do so. We cannot just read and block from a
+ // file, waiting for further output to appear... as this only works on pipes
+ // or sockets. We need a better interface for this whole thing.
+ {
+ std::auto_ptr< std::ostream > output = utils::open_ostream(
+ stdout_target);
+ *output << utils::read_file(result_handle->stdout_file());
+ }
+ {
+ std::auto_ptr< std::ostream > output = utils::open_ostream(
+ stderr_target);
+ *output << utils::read_file(result_handle->stderr_file());
+ }
+
+ INV(result_handle->original_pid() == exec_handle);
+ return result_handle;
+}
+
+
+/// Checks if an interrupt has fired.
+///
+/// Calls to this function should be sprinkled in strategic places through the
+/// code protected by an interrupts_handler object.
+///
+/// This is just a wrapper over signals::check_interrupt() to avoid leaking this
+/// dependency to the caller.
+///
+/// \throw signals::interrupted_error If there has been an interrupt.
+void
+scheduler::scheduler_handle::check_interrupt(void) const
+{
+ _pimpl->generic.check_interrupt();
+}
+
+
+/// Queries the current execution context.
+///
+/// \return The queried context.
+model::context
+scheduler::current_context(void)
+{
+ return model::context(fs::current_path(), utils::getallenv());
+}
+
+
+/// Generates the set of configuration variables for a test program.
+///
+/// \param user_config The configuration variables provided by the user.
+/// \param test_suite The name of the test suite.
+///
+/// \return The mapping of configuration variables for the test program.
+config::properties_map
+scheduler::generate_config(const config::tree& user_config,
+ const std::string& test_suite)
+{
+ config::properties_map props;
+
+ try {
+ props = user_config.all_properties(F("test_suites.%s") % test_suite,
+ true);
+ } catch (const config::unknown_key_error& unused_error) {
+ // Ignore: not all test suites have entries in the configuration.
+ }
+
+ // TODO(jmmv): This is a hack that exists for the ATF interface only, so it
+ // should be moved there.
+ if (user_config.is_set("unprivileged_user")) {
+ const passwd::user& user =
+ user_config.lookup< engine::user_node >("unprivileged_user");
+ props["unprivileged-user"] = user.name;
+ }
+
+ return props;
+}
diff --git a/engine/scheduler.hpp b/engine/scheduler.hpp
new file mode 100644
index 000000000000..24ff0b5a26fc
--- /dev/null
+++ b/engine/scheduler.hpp
@@ -0,0 +1,282 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/scheduler.hpp
+/// Multiprogrammed executor of test related operations.
+///
+/// The scheduler's public interface exposes test cases as "black boxes". The
+/// handling of cleanup routines is completely hidden from the caller and
+/// happens in two cases: first, once a test case completes; and, second, in the
+/// case of abrupt termination due to the reception of a signal.
+///
+/// Hiding cleanup routines from the caller is an attempt to keep the logic of
+/// execution and results handling in a single place. Otherwise, the various
+/// drivers (say run_tests and debug_test) would need to replicate the handling
+/// of this logic, which is tricky in itself (particularly due to signal
+/// handling) and would lead to inconsistencies.
+///
+/// Handling cleanup routines in the manner described above is *incredibly
+/// complicated* (insane, actually) as you will see from the code. The
+/// complexity will bite us in the future (today is 2015-06-26). Switching to a
+/// threads-based implementation would probably simplify the code flow
+/// significantly and allow parallelization of the test case listings in a
+/// reasonable manner, though it depends on whether we can get clean handling of
+/// signals and on whether we could use C++11's std::thread. (Is this a to-do?
+/// Maybe. Maybe not.)
+///
+/// See the documentation in utils/process/executor.hpp for details on
+/// the expected workflow of these classes.
+
+#if !defined(ENGINE_SCHEDULER_HPP)
+#define ENGINE_SCHEDULER_HPP
+
+#include "engine/scheduler_fwd.hpp"
+
+#include <memory>
+#include <set>
+#include <string>
+
+#include "model/context_fwd.hpp"
+#include "model/metadata_fwd.hpp"
+#include "model/test_case_fwd.hpp"
+#include "model/test_program.hpp"
+#include "model/test_result_fwd.hpp"
+#include "utils/config/tree_fwd.hpp"
+#include "utils/datetime_fwd.hpp"
+#include "utils/defs.hpp"
+#include "utils/fs/path_fwd.hpp"
+#include "utils/optional.hpp"
+#include "utils/process/executor_fwd.hpp"
+#include "utils/process/status_fwd.hpp"
+
+namespace engine {
+namespace scheduler {
+
+
+/// Abstract interface of a test program scheduler interface.
+///
+/// This interface defines the test program-specific operations that need to be
+/// invoked at different points during the execution of a given test case. The
+/// scheduler internally instantiates one of these for every test case.
+class interface {
+public:
+ /// Destructor.
+ virtual ~interface() {}
+
+ /// Executes a test program's list operation.
+ ///
+ /// This method is intended to be called within a subprocess and is expected
+ /// to terminate execution either by exec(2)ing the test program or by
+ /// exiting with a failure.
+ ///
+ /// \param test_program The test program to execute.
+ /// \param vars User-provided variables to pass to the test program.
+ virtual void exec_list(const model::test_program& test_program,
+ const utils::config::properties_map& vars)
+ const UTILS_NORETURN = 0;
+
+ /// Computes the test cases list of a test program.
+ ///
+ /// \param status The termination status of the subprocess used to execute
+ /// the exec_test() method or none if the test timed out.
+ /// \param stdout_path Path to the file containing the stdout of the test.
+ /// \param stderr_path Path to the file containing the stderr of the test.
+ ///
+ /// \return A list of test cases.
+ virtual model::test_cases_map parse_list(
+ const utils::optional< utils::process::status >& status,
+ const utils::fs::path& stdout_path,
+ const utils::fs::path& stderr_path) const = 0;
+
+ /// Executes a test case of the test program.
+ ///
+ /// This method is intended to be called within a subprocess and is expected
+ /// to terminate execution either by exec(2)ing the test program or by
+ /// exiting with a failure.
+ ///
+ /// \param test_program The test program to execute.
+ /// \param test_case_name Name of the test case to invoke.
+ /// \param vars User-provided variables to pass to the test program.
+ /// \param control_directory Directory where the interface may place control
+ /// files.
+ virtual void exec_test(const model::test_program& test_program,
+ const std::string& test_case_name,
+ const utils::config::properties_map& vars,
+ const utils::fs::path& control_directory)
+ const UTILS_NORETURN = 0;
+
+ /// Executes a test cleanup routine of the test program.
+ ///
+ /// This method is intended to be called within a subprocess and is expected
+ /// to terminate execution either by exec(2)ing the test program or by
+ /// exiting with a failure.
+ ///
+ /// \param test_program The test program to execute.
+ /// \param test_case_name Name of the test case to invoke.
+ /// \param vars User-provided variables to pass to the test program.
+ /// \param control_directory Directory where the interface may place control
+ /// files.
+ virtual void exec_cleanup(const model::test_program& test_program,
+ const std::string& test_case_name,
+ const utils::config::properties_map& vars,
+ const utils::fs::path& control_directory)
+ const UTILS_NORETURN;
+
+ /// Computes the result of a test case based on its termination status.
+ ///
+ /// \param status The termination status of the subprocess used to execute
+ /// the exec_test() method or none if the test timed out.
+ /// \param control_directory Directory where the interface may have placed
+ /// control files.
+ /// \param stdout_path Path to the file containing the stdout of the test.
+ /// \param stderr_path Path to the file containing the stderr of the test.
+ ///
+ /// \return A test result.
+ virtual model::test_result compute_result(
+ const utils::optional< utils::process::status >& status,
+ const utils::fs::path& control_directory,
+ const utils::fs::path& stdout_path,
+ const utils::fs::path& stderr_path) const = 0;
+};
+
+
+/// Implementation of a test program with lazy loading of test cases.
+class lazy_test_program : public model::test_program {
+ struct impl;
+
+ /// Pointer to the shared internal implementation.
+ std::shared_ptr< impl > _pimpl;
+
+public:
+ lazy_test_program(const std::string&, const utils::fs::path&,
+ const utils::fs::path&, const std::string&,
+ const model::metadata&,
+ const utils::config::tree&,
+ scheduler_handle&);
+
+ const model::test_cases_map& test_cases(void) const;
+};
+
+
+/// Base type containing the results of the execution of a subprocess.
+class result_handle {
+protected:
+ struct bimpl;
+
+private:
+ /// Pointer to internal implementation of the base type.
+ std::shared_ptr< bimpl > _pbimpl;
+
+protected:
+ friend class scheduler_handle;
+ result_handle(std::shared_ptr< bimpl >);
+
+public:
+ virtual ~result_handle(void) = 0;
+
+ void cleanup(void);
+
+ int original_pid(void) const;
+ const utils::datetime::timestamp& start_time() const;
+ const utils::datetime::timestamp& end_time() const;
+ utils::fs::path work_directory(void) const;
+ const utils::fs::path& stdout_file(void) const;
+ const utils::fs::path& stderr_file(void) const;
+};
+
+
+/// Container for all test termination data and accessor to cleanup operations.
+class test_result_handle : public result_handle {
+ struct impl;
+ /// Pointer to internal implementation.
+ std::shared_ptr< impl > _pimpl;
+
+ friend class scheduler_handle;
+ test_result_handle(std::shared_ptr< bimpl >, std::shared_ptr< impl >);
+
+public:
+ ~test_result_handle(void);
+
+ const model::test_program_ptr test_program(void) const;
+ const std::string& test_case_name(void) const;
+ const model::test_result& test_result(void) const;
+};
+
+
+/// Stateful interface to the multiprogrammed execution of tests.
+class scheduler_handle {
+ struct impl;
+ /// Pointer to internal implementation.
+ std::shared_ptr< impl > _pimpl;
+
+ friend scheduler_handle setup(void);
+ scheduler_handle(void);
+
+public:
+ ~scheduler_handle(void);
+
+ const utils::fs::path& root_work_directory(void) const;
+
+ void cleanup(void);
+
+ model::test_cases_map list_tests(const model::test_program*,
+ const utils::config::tree&);
+ exec_handle spawn_test(const model::test_program_ptr,
+ const std::string&,
+ const utils::config::tree&);
+ result_handle_ptr wait_any(void);
+
+ result_handle_ptr debug_test(const model::test_program_ptr,
+ const std::string&,
+ const utils::config::tree&,
+ const utils::fs::path&,
+ const utils::fs::path&);
+
+ void check_interrupt(void) const;
+};
+
+
+extern utils::datetime::delta cleanup_timeout;
+extern utils::datetime::delta list_timeout;
+
+
+void ensure_valid_interface(const std::string&);
+void register_interface(const std::string&, const std::shared_ptr< interface >);
+std::set< std::string > registered_interface_names(void);
+scheduler_handle setup(void);
+
+model::context current_context(void);
+utils::config::properties_map generate_config(const utils::config::tree&,
+ const std::string&);
+
+
+} // namespace scheduler
+} // namespace engine
+
+
+#endif // !defined(ENGINE_SCHEDULER_HPP)
diff --git a/engine/scheduler_fwd.hpp b/engine/scheduler_fwd.hpp
new file mode 100644
index 000000000000..f61b084e5a8d
--- /dev/null
+++ b/engine/scheduler_fwd.hpp
@@ -0,0 +1,61 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/scheduler_fwd.hpp
+/// Forward declarations for engine/scheduler.hpp
+
+#if !defined(ENGINE_SCHEDULER_FWD_HPP)
+#define ENGINE_SCHEDULER_FWD_HPP
+
+#include <memory>
+
+namespace engine {
+namespace scheduler {
+
+
+/// Unique identifier for in-flight execution operations.
+///
+/// TODO(jmmv): Might be worth to drop altogether and just use "int". The type
+/// difference with executor::exec_handle is confusing.
+typedef int exec_handle;
+
+
+class scheduler_handle;
+class interface;
+class result_handle;
+class test_result_handle;
+
+
+/// Pointer to a dynamically-allocated result_handle.
+typedef std::shared_ptr< result_handle > result_handle_ptr;
+
+
+} // namespace scheduler
+} // namespace engine
+
+#endif // !defined(ENGINE_SCHEDULER_FWD_HPP)
diff --git a/engine/scheduler_test.cpp b/engine/scheduler_test.cpp
new file mode 100644
index 000000000000..e144761d8f01
--- /dev/null
+++ b/engine/scheduler_test.cpp
@@ -0,0 +1,1242 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/scheduler.hpp"
+
+extern "C" {
+#include <sys/types.h>
+
+#include <signal.h>
+#include <unistd.h>
+}
+
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <string>
+
+#include <atf-c++.hpp>
+
+#include "engine/config.hpp"
+#include "engine/exceptions.hpp"
+#include "model/context.hpp"
+#include "model/metadata.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "model/test_result.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/datetime.hpp"
+#include "utils/defs.hpp"
+#include "utils/env.hpp"
+#include "utils/format/containers.ipp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/optional.ipp"
+#include "utils/passwd.hpp"
+#include "utils/process/status.hpp"
+#include "utils/sanity.hpp"
+#include "utils/stacktrace.hpp"
+#include "utils/stream.hpp"
+#include "utils/test_utils.ipp"
+#include "utils/text/exceptions.hpp"
+#include "utils/text/operations.ipp"
+
+namespace config = utils::config;
+namespace datetime = utils::datetime;
+namespace fs = utils::fs;
+namespace passwd = utils::passwd;
+namespace process = utils::process;
+namespace scheduler = engine::scheduler;
+namespace text = utils::text;
+
+using utils::none;
+using utils::optional;
+
+
+namespace {
+
+
+/// Checks if a string starts with a prefix.
+///
+/// \param str The string to be tested.
+/// \param prefix The prefix to look for.
+///
+/// \return True if the string is prefixed as specified.
+static bool
+starts_with(const std::string& str, const std::string& prefix)
+{
+ return (str.length() >= prefix.length() &&
+ str.substr(0, prefix.length()) == prefix);
+}
+
+
+/// Strips a prefix from a string and converts the rest to an integer.
+///
+/// \param str The string to be tested.
+/// \param prefix The prefix to strip from the string.
+///
+/// \return The part of the string after the prefix converted to an integer.
+static int
+suffix_to_int(const std::string& str, const std::string& prefix)
+{
+ PRE(starts_with(str, prefix));
+ try {
+ return text::to_type< int >(str.substr(prefix.length()));
+ } catch (const text::value_error& error) {
+ std::cerr << F("Failed: %s\n") % error.what();
+ std::abort();
+ }
+}
+
+
+/// Mock interface definition for testing.
+///
+/// This scheduler interface does not execute external binaries. It is designed
+/// to simulate the scheduler of various programs with different exit statuses.
+class mock_interface : public scheduler::interface {
+ /// Executes the subprocess simulating an exec.
+ ///
+ /// This is just a simple wrapper over _exit(2) because we cannot use
+ /// std::exit on exit from this mock interface. The reason is that we do
+ /// not want to invoke any destructors as otherwise we'd clear up the global
+ /// scheduler state by mistake. This wouldn't be a major problem if it
+ /// wasn't because doing so deletes on-disk files and we want to leave them
+ /// in place so that the parent process can test for them!
+ ///
+ /// \param exit_code Exit code.
+ void
+ do_exit(const int exit_code) const UTILS_NORETURN
+ {
+ std::cout.flush();
+ std::cerr.flush();
+ ::_exit(exit_code);
+ }
+
+ /// Executes a test case that creates various files and then fails.
+ void
+ exec_create_files_and_fail(void) const UTILS_NORETURN
+ {
+ std::cerr << "This should not be clobbered\n";
+ atf::utils::create_file("first file", "");
+ atf::utils::create_file("second-file", "");
+ fs::mkdir_p(fs::path("dir1/dir2"), 0755);
+ ::kill(::getpid(), SIGTERM);
+ std::abort();
+ }
+
+ /// Executes a test case that deletes all files in the current directory.
+ ///
+ /// This is intended to validate that the test runs in an empty directory,
+ /// separate from any control files that the scheduler may have created.
+ void
+ exec_delete_all(void) const UTILS_NORETURN
+ {
+ const int exit_code = ::system("rm *") == -1
+ ? EXIT_FAILURE : EXIT_SUCCESS;
+
+ // Recreate our own cookie.
+ atf::utils::create_file("exec_test_was_called", "");
+
+ do_exit(exit_code);
+ }
+
+ /// Executes a test case that returns a specific exit code.
+ ///
+ /// \param exit_code Exit status to terminate the program with.
+ void
+ exec_exit(const int exit_code) const UTILS_NORETURN
+ {
+ do_exit(exit_code);
+ }
+
+ /// Executes a test case that just fails.
+ void
+ exec_fail(void) const UTILS_NORETURN
+ {
+ std::cerr << "This should not be clobbered\n";
+ ::kill(::getpid(), SIGTERM);
+ std::abort();
+ }
+
+ /// Executes a test case that prints all input parameters to the functor.
+ ///
+ /// \param test_program The test program to execute.
+ /// \param test_case_name Name of the test case to invoke, which must be a
+ /// number.
+ /// \param vars User-provided variables to pass to the test program.
+ void
+ exec_print_params(const model::test_program& test_program,
+ const std::string& test_case_name,
+ const config::properties_map& vars) const
+ UTILS_NORETURN
+ {
+ std::cout << F("Test program: %s\n") % test_program.relative_path();
+ std::cout << F("Test case: %s\n") % test_case_name;
+ for (config::properties_map::const_iterator iter = vars.begin();
+ iter != vars.end(); ++iter) {
+ std::cout << F("%s=%s\n") % (*iter).first % (*iter).second;
+ }
+
+ std::cerr << F("stderr: %s\n") % test_case_name;
+
+ do_exit(EXIT_SUCCESS);
+ }
+
+public:
+ /// Executes a test program's list operation.
+ ///
+ /// This method is intended to be called within a subprocess and is expected
+ /// to terminate execution either by exec(2)ing the test program or by
+ /// exiting with a failure.
+ ///
+ /// \param test_program The test program to execute.
+ /// \param vars User-provided variables to pass to the test program.
+ void
+ exec_list(const model::test_program& test_program,
+ const config::properties_map& vars)
+ const UTILS_NORETURN
+ {
+ const std::string name = test_program.absolute_path().leaf_name();
+
+ std::cerr << name;
+ std::cerr.flush();
+ if (name == "check_i_exist") {
+ if (fs::exists(test_program.absolute_path())) {
+ std::cout << "found\n";
+ do_exit(EXIT_SUCCESS);
+ } else {
+ std::cout << "not_found\n";
+ do_exit(EXIT_FAILURE);
+ }
+ } else if (name == "empty") {
+ do_exit(EXIT_SUCCESS);
+ } else if (name == "misbehave") {
+ utils::abort_without_coredump();
+ } else if (name == "timeout") {
+ std::cout << "sleeping\n";
+ std::cout.flush();
+ ::sleep(100);
+ utils::abort_without_coredump();
+ } else if (name == "vars") {
+ for (config::properties_map::const_iterator iter = vars.begin();
+ iter != vars.end(); ++iter) {
+ std::cout << F("%s_%s\n") % (*iter).first % (*iter).second;
+ }
+ do_exit(15);
+ } else {
+ std::abort();
+ }
+ }
+
+ /// Computes the test cases list of a test program.
+ ///
+ /// \param status The termination status of the subprocess used to execute
+ /// the exec_test() method or none if the test timed out.
+ /// \param stdout_path Path to the file containing the stdout of the test.
+ /// \param stderr_path Path to the file containing the stderr of the test.
+ ///
+ /// \return A list of test cases.
+ model::test_cases_map
+ parse_list(const optional< process::status >& status,
+ const fs::path& stdout_path,
+ const fs::path& stderr_path) const
+ {
+ const std::string name = utils::read_file(stderr_path);
+ if (name == "check_i_exist") {
+ ATF_REQUIRE(status.get().exited());
+ ATF_REQUIRE_EQ(EXIT_SUCCESS, status.get().exitstatus());
+ } else if (name == "empty") {
+ ATF_REQUIRE(status.get().exited());
+ ATF_REQUIRE_EQ(EXIT_SUCCESS, status.get().exitstatus());
+ } else if (name == "misbehave") {
+ throw std::runtime_error("misbehaved in parse_list");
+ } else if (name == "timeout") {
+ ATF_REQUIRE(!status);
+ } else if (name == "vars") {
+ ATF_REQUIRE(status.get().exited());
+ ATF_REQUIRE_EQ(15, status.get().exitstatus());
+ } else {
+ ATF_FAIL("Invalid stderr contents; got " + name);
+ }
+
+ model::test_cases_map_builder test_cases_builder;
+
+ std::ifstream input(stdout_path.c_str());
+ ATF_REQUIRE(input);
+ std::string line;
+ while (std::getline(input, line).good()) {
+ test_cases_builder.add(line);
+ }
+
+ return test_cases_builder.build();
+ }
+
+ /// Executes a test case of the test program.
+ ///
+ /// This method is intended to be called within a subprocess and is expected
+ /// to terminate execution either by exec(2)ing the test program or by
+ /// exiting with a failure.
+ ///
+ /// \param test_program The test program to execute.
+ /// \param test_case_name Name of the test case to invoke.
+ /// \param vars User-provided variables to pass to the test program.
+ /// \param control_directory Directory where the interface may place control
+ /// files.
+ void
+ exec_test(const model::test_program& test_program,
+ const std::string& test_case_name,
+ const config::properties_map& vars,
+ const fs::path& control_directory) const
+ {
+ const fs::path cookie = control_directory / "exec_test_was_called";
+ std::ofstream control_file(cookie.c_str());
+ if (!control_file) {
+ std::cerr << "Failed to create " << cookie << '\n';
+ std::abort();
+ }
+ control_file << test_case_name;
+ control_file.close();
+
+ if (test_case_name == "check_i_exist") {
+ do_exit(fs::exists(test_program.absolute_path()) ? 0 : 1);
+ } else if (starts_with(test_case_name, "cleanup_timeout")) {
+ exec_exit(EXIT_SUCCESS);
+ } else if (starts_with(test_case_name, "create_files_and_fail")) {
+ exec_create_files_and_fail();
+ } else if (test_case_name == "delete_all") {
+ exec_delete_all();
+ } else if (starts_with(test_case_name, "exit ")) {
+ exec_exit(suffix_to_int(test_case_name, "exit "));
+ } else if (starts_with(test_case_name, "fail")) {
+ exec_fail();
+ } else if (starts_with(test_case_name, "fail_body_fail_cleanup")) {
+ exec_fail();
+ } else if (starts_with(test_case_name, "fail_body_pass_cleanup")) {
+ exec_fail();
+ } else if (starts_with(test_case_name, "pass_body_fail_cleanup")) {
+ exec_exit(EXIT_SUCCESS);
+ } else if (starts_with(test_case_name, "print_params")) {
+ exec_print_params(test_program, test_case_name, vars);
+ } else if (starts_with(test_case_name, "skip_body_pass_cleanup")) {
+ exec_exit(EXIT_SUCCESS);
+ } else {
+ std::cerr << "Unknown test case " << test_case_name << '\n';
+ std::abort();
+ }
+ }
+
+ /// Executes a test cleanup routine of the test program.
+ ///
+ /// This method is intended to be called within a subprocess and is expected
+ /// to terminate execution either by exec(2)ing the test program or by
+ /// exiting with a failure.
+ ///
+ /// \param test_case_name Name of the test case to invoke.
+ void
+ exec_cleanup(const model::test_program& /* test_program */,
+ const std::string& test_case_name,
+ const config::properties_map& /* vars */,
+ const fs::path& /* control_directory */) const
+ {
+ std::cout << "exec_cleanup was called\n";
+ std::cout.flush();
+
+ if (starts_with(test_case_name, "cleanup_timeout")) {
+ ::sleep(100);
+ std::abort();
+ } else if (starts_with(test_case_name, "fail_body_fail_cleanup")) {
+ exec_fail();
+ } else if (starts_with(test_case_name, "fail_body_pass_cleanup")) {
+ exec_exit(EXIT_SUCCESS);
+ } else if (starts_with(test_case_name, "pass_body_fail_cleanup")) {
+ exec_fail();
+ } else if (starts_with(test_case_name, "skip_body_pass_cleanup")) {
+ exec_exit(EXIT_SUCCESS);
+ } else {
+ std::cerr << "Should not have been called for a test without "
+ "a cleanup routine" << '\n';
+ std::abort();
+ }
+ }
+
+ /// Computes the result of a test case based on its termination status.
+ ///
+ /// \param status The termination status of the subprocess used to execute
+ /// the exec_test() method or none if the test timed out.
+ /// \param control_directory Path to the directory where the interface may
+ /// have placed control files.
+ /// \param stdout_path Path to the file containing the stdout of the test.
+ /// \param stderr_path Path to the file containing the stderr of the test.
+ ///
+ /// \return A test result.
+ model::test_result
+ compute_result(const optional< process::status >& status,
+ const fs::path& control_directory,
+ const fs::path& stdout_path,
+ const fs::path& stderr_path) const
+ {
+ // Do not use any ATF_* macros here. Some of the tests below invoke
+ // this code in a subprocess, and terminating such subprocess due to a
+ // failed ATF_* macro yields mysterious failures that are incredibly
+ // hard to debug. (Case in point: the signal_handling test is racy by
+ // nature, and the test run by exec_test() above may not have created
+ // the cookie we expect below. We don't want to "silently" exit if the
+ // file is not there.)
+
+ if (!status) {
+ return model::test_result(model::test_result_broken,
+ "Timed out");
+ }
+
+ if (status.get().exited()) {
+ // Only sanity-check the work directory-related parameters in case
+ // of a clean exit. In all other cases, there is no guarantee that
+ // these were ever created.
+ const fs::path cookie = control_directory / "exec_test_was_called";
+ if (!atf::utils::file_exists(cookie.str())) {
+ return model::test_result(
+ model::test_result_broken,
+ "compute_result's control_directory does not seem to point "
+ "to the right location");
+ }
+ const std::string test_case_name = utils::read_file(cookie);
+
+ if (!atf::utils::file_exists(stdout_path.str())) {
+ return model::test_result(
+ model::test_result_broken,
+ "compute_result's stdout_path does not exist");
+ }
+ if (!atf::utils::file_exists(stderr_path.str())) {
+ return model::test_result(
+ model::test_result_broken,
+ "compute_result's stderr_path does not exist");
+ }
+
+ if (test_case_name == "skip_body_pass_cleanup") {
+ return model::test_result(
+ model::test_result_skipped,
+ F("Exit %s") % status.get().exitstatus());
+ } else {
+ return model::test_result(
+ model::test_result_passed,
+ F("Exit %s") % status.get().exitstatus());
+ }
+ } else {
+ return model::test_result(
+ model::test_result_failed,
+ F("Signal %s") % status.get().termsig());
+ }
+ }
+};
+
+
+} // anonymous namespace
+
+
+/// Runs list_tests on the scheduler and returns the results.
+///
+/// \param test_name The name of the test supported by our exec_list function.
+/// \param user_config Optional user settings for the test.
+///
+/// \return The loaded list of test cases.
+static model::test_cases_map
+check_integration_list(const char* test_name, const fs::path root,
+ const config::tree& user_config = engine::empty_config())
+{
+ const model::test_program program = model::test_program_builder(
+ "mock", fs::path(test_name), root, "the-suite")
+ .build();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+ const model::test_cases_map test_cases = handle.list_tests(&program,
+ user_config);
+ handle.cleanup();
+
+ return test_cases;
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_some);
+ATF_TEST_CASE_BODY(integration__list_some)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.first", "test");
+ user_config.set_string("test_suites.the-suite.second", "TEST");
+ user_config.set_string("test_suites.abc.unused", "unused");
+
+ const model::test_cases_map test_cases = check_integration_list(
+ "vars", fs::path("."), user_config);
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("first_test").add("second_TEST").build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_check_paths);
+ATF_TEST_CASE_BODY(integration__list_check_paths)
+{
+ fs::mkdir_p(fs::path("dir1/dir2/dir3"), 0755);
+ atf::utils::create_file("dir1/dir2/dir3/check_i_exist", "");
+
+ const model::test_cases_map test_cases = check_integration_list(
+ "dir2/dir3/check_i_exist", fs::path("dir1"));
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("found").build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_timeout);
+ATF_TEST_CASE_BODY(integration__list_timeout)
+{
+ scheduler::list_timeout = datetime::delta(1, 0);
+ const model::test_cases_map test_cases = check_integration_list(
+ "timeout", fs::path("."));
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("sleeping").build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_fail);
+ATF_TEST_CASE_BODY(integration__list_fail)
+{
+ const model::test_cases_map test_cases = check_integration_list(
+ "misbehave", fs::path("."));
+
+ ATF_REQUIRE_EQ(1, test_cases.size());
+ const model::test_case& test_case = test_cases.begin()->second;
+ ATF_REQUIRE_EQ("__test_cases_list__", test_case.name());
+ ATF_REQUIRE(test_case.fake_result());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_broken,
+ "misbehaved in parse_list"),
+ test_case.fake_result().get());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_empty);
+ATF_TEST_CASE_BODY(integration__list_empty)
+{
+ const model::test_cases_map test_cases = check_integration_list(
+ "empty", fs::path("."));
+
+ ATF_REQUIRE_EQ(1, test_cases.size());
+ const model::test_case& test_case = test_cases.begin()->second;
+ ATF_REQUIRE_EQ("__test_cases_list__", test_case.name());
+ ATF_REQUIRE(test_case.fake_result());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_broken,
+ "Empty test cases list"),
+ test_case.fake_result().get());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__run_one);
+ATF_TEST_CASE_BODY(integration__run_one)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("exit 41").build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ const scheduler::exec_handle exec_handle = handle.spawn_test(
+ program, "exit 41", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(exec_handle, result_handle->original_pid());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 41"),
+ test_result_handle->test_result());
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__run_many);
+ATF_TEST_CASE_BODY(integration__run_many)
+{
+ static const std::size_t num_test_programs = 30;
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ // We mess around with the "current time" below, so make sure the tests do
+ // not spuriously exceed their deadline by bumping it to a large number.
+ const model::metadata infinite_timeout = model::metadata_builder()
+ .set_timeout(datetime::delta(1000000L, 0)).build();
+
+ std::size_t total_tests = 0;
+ std::map< scheduler::exec_handle, model::test_program_ptr >
+ exp_test_programs;
+ std::map< scheduler::exec_handle, std::string > exp_test_case_names;
+ std::map< scheduler::exec_handle, datetime::timestamp > exp_start_times;
+ std::map< scheduler::exec_handle, int > exp_exit_statuses;
+ for (std::size_t i = 0; i < num_test_programs; ++i) {
+ const std::string test_case_0 = F("exit %s") % (i * 3 + 0);
+ const std::string test_case_1 = F("exit %s") % (i * 3 + 1);
+ const std::string test_case_2 = F("exit %s") % (i * 3 + 2);
+
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path(F("program-%s") % i),
+ fs::current_path(), "the-suite")
+ .set_metadata(infinite_timeout)
+ .add_test_case(test_case_0)
+ .add_test_case(test_case_1)
+ .add_test_case(test_case_2)
+ .build_ptr();
+
+ const datetime::timestamp start_time = datetime::timestamp::from_values(
+ 2014, 12, 8, 9, 40, 0, i);
+
+ scheduler::exec_handle exec_handle;
+
+ datetime::set_mock_now(start_time);
+ exec_handle = handle.spawn_test(program, test_case_0, user_config);
+ exp_test_programs.insert(std::make_pair(exec_handle, program));
+ exp_test_case_names.insert(std::make_pair(exec_handle, test_case_0));
+ exp_start_times.insert(std::make_pair(exec_handle, start_time));
+ exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3));
+ ++total_tests;
+
+ datetime::set_mock_now(start_time);
+ exec_handle = handle.spawn_test(program, test_case_1, user_config);
+ exp_test_programs.insert(std::make_pair(exec_handle, program));
+ exp_test_case_names.insert(std::make_pair(exec_handle, test_case_1));
+ exp_start_times.insert(std::make_pair(exec_handle, start_time));
+ exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3 + 1));
+ ++total_tests;
+
+ datetime::set_mock_now(start_time);
+ exec_handle = handle.spawn_test(program, test_case_2, user_config);
+ exp_test_programs.insert(std::make_pair(exec_handle, program));
+ exp_test_case_names.insert(std::make_pair(exec_handle, test_case_2));
+ exp_start_times.insert(std::make_pair(exec_handle, start_time));
+ exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3 + 2));
+ ++total_tests;
+ }
+
+ for (std::size_t i = 0; i < total_tests; ++i) {
+ const datetime::timestamp end_time = datetime::timestamp::from_values(
+ 2014, 12, 8, 9, 50, 10, i);
+ datetime::set_mock_now(end_time);
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+
+ const scheduler::exec_handle exec_handle =
+ result_handle->original_pid();
+
+ const model::test_program_ptr test_program = exp_test_programs.find(
+ exec_handle)->second;
+ const std::string& test_case_name = exp_test_case_names.find(
+ exec_handle)->second;
+ const datetime::timestamp& start_time = exp_start_times.find(
+ exec_handle)->second;
+ const int exit_status = exp_exit_statuses.find(exec_handle)->second;
+
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed,
+ F("Exit %s") % exit_status),
+ test_result_handle->test_result());
+
+ ATF_REQUIRE_EQ(test_program, test_result_handle->test_program());
+ ATF_REQUIRE_EQ(test_case_name, test_result_handle->test_case_name());
+
+ ATF_REQUIRE_EQ(start_time, result_handle->start_time());
+ ATF_REQUIRE_EQ(end_time, result_handle->end_time());
+
+ result_handle->cleanup();
+
+ ATF_REQUIRE(!atf::utils::file_exists(
+ result_handle->stdout_file().str()));
+ ATF_REQUIRE(!atf::utils::file_exists(
+ result_handle->stderr_file().str()));
+ ATF_REQUIRE(!atf::utils::file_exists(
+ result_handle->work_directory().str()));
+
+ result_handle.reset();
+ }
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__run_check_paths);
+ATF_TEST_CASE_BODY(integration__run_check_paths)
+{
+ fs::mkdir_p(fs::path("dir1/dir2/dir3"), 0755);
+ atf::utils::create_file("dir1/dir2/dir3/program", "");
+
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("dir2/dir3/program"), fs::path("dir1"), "the-suite")
+ .add_test_case("check_i_exist").build_ptr();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "check_i_exist", engine::default_config());
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
+ test_result_handle->test_result());
+
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__parameters_and_output);
+ATF_TEST_CASE_BODY(integration__parameters_and_output)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("print_params").build_ptr();
+
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.one", "first variable");
+ user_config.set_string("test_suites.the-suite.two", "second variable");
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ const scheduler::exec_handle exec_handle = handle.spawn_test(
+ program, "print_params", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+
+ ATF_REQUIRE_EQ(exec_handle, result_handle->original_pid());
+ ATF_REQUIRE_EQ(program, test_result_handle->test_program());
+ ATF_REQUIRE_EQ("print_params", test_result_handle->test_case_name());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
+ test_result_handle->test_result());
+
+ const fs::path stdout_file = result_handle->stdout_file();
+ ATF_REQUIRE(atf::utils::compare_file(
+ stdout_file.str(),
+ "Test program: the-program\n"
+ "Test case: print_params\n"
+ "one=first variable\n"
+ "two=second variable\n"));
+ const fs::path stderr_file = result_handle->stderr_file();
+ ATF_REQUIRE(atf::utils::compare_file(
+ stderr_file.str(), "stderr: print_params\n"));
+
+ result_handle->cleanup();
+ ATF_REQUIRE(!fs::exists(stdout_file));
+ ATF_REQUIRE(!fs::exists(stderr_file));
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__fake_result);
+ATF_TEST_CASE_BODY(integration__fake_result)
+{
+ const model::test_result fake_result(model::test_result_skipped,
+ "Some fake details");
+
+ model::test_cases_map test_cases;
+ test_cases.insert(model::test_cases_map::value_type(
+ "__fake__", model::test_case("__fake__", "ABC", fake_result)));
+
+ const model::test_program_ptr program(new model::test_program(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite",
+ model::metadata_builder().build(), test_cases));
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "__fake__", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(fake_result, test_result_handle->test_result());
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__head_skips);
+ATF_TEST_CASE_BODY(integration__cleanup__head_skips)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("skip_me",
+ model::metadata_builder()
+ .add_required_config("variable-that-does-not-exist")
+ .set_has_cleanup(true)
+ .build())
+ .build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "skip_me", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(model::test_result(
+ model::test_result_skipped,
+ "Required configuration property "
+ "'variable-that-does-not-exist' not defined"),
+ test_result_handle->test_result());
+ ATF_REQUIRE(!atf::utils::grep_file("exec_cleanup was called",
+ result_handle->stdout_file().str()));
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+/// Runs a test to verify the behavior of cleanup routines.
+///
+/// \param test_case The name of the test case to invoke.
+/// \param exp_result The expected test result of the execution.
+static void
+do_cleanup_test(const char* test_case,
+ const model::test_result& exp_result)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case(test_case)
+ .set_metadata(model::metadata_builder().set_has_cleanup(true).build())
+ .build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, test_case, user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result());
+ ATF_REQUIRE(atf::utils::compare_file(
+ result_handle->stdout_file().str(),
+ "exec_cleanup was called\n"));
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_skips);
+ATF_TEST_CASE_BODY(integration__cleanup__body_skips)
+{
+ do_cleanup_test(
+ "skip_body_pass_cleanup",
+ model::test_result(model::test_result_skipped, "Exit 0"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_bad__cleanup_ok);
+ATF_TEST_CASE_BODY(integration__cleanup__body_bad__cleanup_ok)
+{
+ do_cleanup_test(
+ "fail_body_pass_cleanup",
+ model::test_result(model::test_result_failed, "Signal 15"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_ok__cleanup_bad);
+ATF_TEST_CASE_BODY(integration__cleanup__body_ok__cleanup_bad)
+{
+ do_cleanup_test(
+ "pass_body_fail_cleanup",
+ model::test_result(model::test_result_broken, "Test case cleanup "
+ "did not terminate successfully"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_bad__cleanup_bad);
+ATF_TEST_CASE_BODY(integration__cleanup__body_bad__cleanup_bad)
+{
+ do_cleanup_test(
+ "fail_body_fail_cleanup",
+ model::test_result(model::test_result_failed, "Signal 15"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__timeout);
+ATF_TEST_CASE_BODY(integration__cleanup__timeout)
+{
+ scheduler::cleanup_timeout = datetime::delta(1, 0);
+ do_cleanup_test(
+ "cleanup_timeout",
+ model::test_result(model::test_result_broken, "Test case cleanup "
+ "timed out"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__check_requirements);
+ATF_TEST_CASE_BODY(integration__check_requirements)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("exit 12")
+ .set_metadata(model::metadata_builder()
+ .add_required_config("abcde").build())
+ .build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "exit 12", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(model::test_result(
+ model::test_result_skipped,
+ "Required configuration property 'abcde' not defined"),
+ test_result_handle->test_result());
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__stacktrace);
+ATF_TEST_CASE_BODY(integration__stacktrace)
+{
+ utils::prepare_coredump_test(this);
+
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("unknown-dumps-core").build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "unknown-dumps-core", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_failed,
+ F("Signal %s") % SIGABRT),
+ test_result_handle->test_result());
+ ATF_REQUIRE(!atf::utils::grep_file("attempting to gather stack trace",
+ result_handle->stdout_file().str()));
+ ATF_REQUIRE( atf::utils::grep_file("attempting to gather stack trace",
+ result_handle->stderr_file().str()));
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+/// Runs a test to verify the dumping of the list of existing files on failure.
+///
+/// \param test_case The name of the test case to invoke.
+/// \param exp_stderr Expected contents of stderr.
+static void
+do_check_list_files_on_failure(const char* test_case, const char* exp_stderr)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case(test_case).build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, test_case, user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ atf::utils::cat_file(result_handle->stdout_file().str(), "child stdout: ");
+ ATF_REQUIRE(atf::utils::compare_file(result_handle->stdout_file().str(),
+ ""));
+ atf::utils::cat_file(result_handle->stderr_file().str(), "child stderr: ");
+ ATF_REQUIRE(atf::utils::compare_file(result_handle->stderr_file().str(),
+ exp_stderr));
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_files_on_failure__none);
+ATF_TEST_CASE_BODY(integration__list_files_on_failure__none)
+{
+ do_check_list_files_on_failure("fail", "This should not be clobbered\n");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_files_on_failure__some);
+ATF_TEST_CASE_BODY(integration__list_files_on_failure__some)
+{
+ do_check_list_files_on_failure(
+ "create_files_and_fail",
+ "This should not be clobbered\n"
+ "Files left in work directory after failure: "
+ "dir1, first file, second-file\n");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__prevent_clobbering_control_files);
+ATF_TEST_CASE_BODY(integration__prevent_clobbering_control_files)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("delete_all").build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "delete_all", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
+ test_result_handle->test_result());
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(debug_test);
+ATF_TEST_CASE_BODY(debug_test)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("print_params").build_ptr();
+
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.one", "first variable");
+ user_config.set_string("test_suites.the-suite.two", "second variable");
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ const fs::path stdout_file("custom-stdout.txt");
+ const fs::path stderr_file("custom-stderr.txt");
+
+ scheduler::result_handle_ptr result_handle = handle.debug_test(
+ program, "print_params", user_config, stdout_file, stderr_file);
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+
+ ATF_REQUIRE_EQ(program, test_result_handle->test_program());
+ ATF_REQUIRE_EQ("print_params", test_result_handle->test_case_name());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
+ test_result_handle->test_result());
+
+ // The original output went to a file. It's only an artifact of
+ // debug_test() that we later get a copy in our own files.
+ ATF_REQUIRE(stdout_file != result_handle->stdout_file());
+ ATF_REQUIRE(stderr_file != result_handle->stderr_file());
+
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+
+ ATF_REQUIRE(atf::utils::compare_file(
+ stdout_file.str(),
+ "Test program: the-program\n"
+ "Test case: print_params\n"
+ "one=first variable\n"
+ "two=second variable\n"));
+ ATF_REQUIRE(atf::utils::compare_file(
+ stderr_file.str(), "stderr: print_params\n"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(ensure_valid_interface);
+ATF_TEST_CASE_BODY(ensure_valid_interface)
+{
+ scheduler::ensure_valid_interface("mock");
+
+ ATF_REQUIRE_THROW_RE(engine::error, "Unsupported test interface 'mock2'",
+ scheduler::ensure_valid_interface("mock2"));
+ scheduler::register_interface(
+ "mock2", std::shared_ptr< scheduler::interface >(new mock_interface()));
+ scheduler::ensure_valid_interface("mock2");
+
+ // Standard interfaces should not be present unless registered.
+ ATF_REQUIRE_THROW_RE(engine::error, "Unsupported test interface 'plain'",
+ scheduler::ensure_valid_interface("plain"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(registered_interface_names);
+ATF_TEST_CASE_BODY(registered_interface_names)
+{
+ std::set< std::string > exp_names;
+
+ exp_names.insert("mock");
+ ATF_REQUIRE_EQ(exp_names, scheduler::registered_interface_names());
+
+ scheduler::register_interface(
+ "mock2", std::shared_ptr< scheduler::interface >(new mock_interface()));
+ exp_names.insert("mock2");
+ ATF_REQUIRE_EQ(exp_names, scheduler::registered_interface_names());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(current_context);
+ATF_TEST_CASE_BODY(current_context)
+{
+ const model::context context = scheduler::current_context();
+ ATF_REQUIRE_EQ(fs::current_path(), context.cwd());
+ ATF_REQUIRE(utils::getallenv() == context.env());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(generate_config__empty);
+ATF_TEST_CASE_BODY(generate_config__empty)
+{
+ const config::tree user_config = engine::empty_config();
+
+ const config::properties_map exp_props;
+
+ ATF_REQUIRE_EQ(exp_props,
+ scheduler::generate_config(user_config, "missing"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(generate_config__no_matches);
+ATF_TEST_CASE_BODY(generate_config__no_matches)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("architecture", "foo");
+ user_config.set_string("test_suites.one.var1", "value 1");
+
+ const config::properties_map exp_props;
+
+ ATF_REQUIRE_EQ(exp_props,
+ scheduler::generate_config(user_config, "two"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(generate_config__some_matches);
+ATF_TEST_CASE_BODY(generate_config__some_matches)
+{
+ std::vector< passwd::user > mock_users;
+ mock_users.push_back(passwd::user("nobody", 1234, 5678));
+ passwd::set_mock_users_for_testing(mock_users);
+
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("architecture", "foo");
+ user_config.set_string("unprivileged_user", "nobody");
+ user_config.set_string("test_suites.one.var1", "value 1");
+ user_config.set_string("test_suites.two.var2", "value 2");
+
+ config::properties_map exp_props;
+ exp_props["unprivileged-user"] = "nobody";
+ exp_props["var1"] = "value 1";
+
+ ATF_REQUIRE_EQ(exp_props,
+ scheduler::generate_config(user_config, "one"));
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ scheduler::register_interface(
+ "mock", std::shared_ptr< scheduler::interface >(new mock_interface()));
+
+ ATF_ADD_TEST_CASE(tcs, integration__list_some);
+ ATF_ADD_TEST_CASE(tcs, integration__list_check_paths);
+ ATF_ADD_TEST_CASE(tcs, integration__list_timeout);
+ ATF_ADD_TEST_CASE(tcs, integration__list_fail);
+ ATF_ADD_TEST_CASE(tcs, integration__list_empty);
+
+ ATF_ADD_TEST_CASE(tcs, integration__run_one);
+ ATF_ADD_TEST_CASE(tcs, integration__run_many);
+
+ ATF_ADD_TEST_CASE(tcs, integration__run_check_paths);
+ ATF_ADD_TEST_CASE(tcs, integration__parameters_and_output);
+
+ ATF_ADD_TEST_CASE(tcs, integration__fake_result);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__head_skips);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_skips);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_ok__cleanup_bad);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_bad__cleanup_ok);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_bad__cleanup_bad);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__timeout);
+ ATF_ADD_TEST_CASE(tcs, integration__check_requirements);
+ ATF_ADD_TEST_CASE(tcs, integration__stacktrace);
+ ATF_ADD_TEST_CASE(tcs, integration__list_files_on_failure__none);
+ ATF_ADD_TEST_CASE(tcs, integration__list_files_on_failure__some);
+ ATF_ADD_TEST_CASE(tcs, integration__prevent_clobbering_control_files);
+
+ ATF_ADD_TEST_CASE(tcs, debug_test);
+
+ ATF_ADD_TEST_CASE(tcs, ensure_valid_interface);
+ ATF_ADD_TEST_CASE(tcs, registered_interface_names);
+
+ ATF_ADD_TEST_CASE(tcs, current_context);
+
+ ATF_ADD_TEST_CASE(tcs, generate_config__empty);
+ ATF_ADD_TEST_CASE(tcs, generate_config__no_matches);
+ ATF_ADD_TEST_CASE(tcs, generate_config__some_matches);
+}
diff --git a/engine/tap.cpp b/engine/tap.cpp
new file mode 100644
index 000000000000..85e23857f5b7
--- /dev/null
+++ b/engine/tap.cpp
@@ -0,0 +1,191 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/tap.hpp"
+
+extern "C" {
+#include <unistd.h>
+}
+
+#include <cstdlib>
+
+#include "engine/exceptions.hpp"
+#include "engine/tap_parser.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "model/test_result.hpp"
+#include "utils/defs.hpp"
+#include "utils/env.hpp"
+#include "utils/format/macros.hpp"
+#include "utils/optional.ipp"
+#include "utils/process/operations.hpp"
+#include "utils/process/status.hpp"
+#include "utils/sanity.hpp"
+
+namespace config = utils::config;
+namespace fs = utils::fs;
+namespace process = utils::process;
+
+using utils::optional;
+
+
+namespace {
+
+
+/// Computes the result of a TAP test program termination.
+///
+/// Timeouts and bad TAP data must be handled by the caller. Here we assume
+/// that we have been able to successfully parse the TAP output.
+///
+/// \param summary Parsed TAP data for the test program.
+/// \param status Exit status of the test program.
+///
+/// \return A test result.
+static model::test_result
+tap_to_result(const engine::tap_summary& summary,
+ const process::status& status)
+{
+ if (summary.bailed_out()) {
+ return model::test_result(model::test_result_failed, "Bailed out");
+ }
+
+ if (summary.plan() == engine::all_skipped_plan) {
+ return model::test_result(model::test_result_skipped,
+ summary.all_skipped_reason());
+ }
+
+ if (summary.not_ok_count() == 0) {
+ if (status.exitstatus() == EXIT_SUCCESS) {
+ return model::test_result(model::test_result_passed);
+ } else {
+ return model::test_result(
+ model::test_result_broken,
+ F("Dubious test program: reported all tests as passed "
+ "but returned exit code %s") % status.exitstatus());
+ }
+ } else {
+ const std::size_t total = summary.ok_count() + summary.not_ok_count();
+ return model::test_result(model::test_result_failed,
+ F("%s of %s tests failed") %
+ summary.not_ok_count() % total);
+ }
+}
+
+
+} // anonymous namespace
+
+
+/// Executes a test program's list operation.
+///
+/// This method is intended to be called within a subprocess and is expected
+/// to terminate execution either by exec(2)ing the test program or by
+/// exiting with a failure.
+void
+engine::tap_interface::exec_list(
+ const model::test_program& /* test_program */,
+ const config::properties_map& /* vars */) const
+{
+ ::_exit(EXIT_SUCCESS);
+}
+
+
+/// Computes the test cases list of a test program.
+///
+/// \return A list of test cases.
+model::test_cases_map
+engine::tap_interface::parse_list(
+ const optional< process::status >& /* status */,
+ const fs::path& /* stdout_path */,
+ const fs::path& /* stderr_path */) const
+{
+ return model::test_cases_map_builder().add("main").build();
+}
+
+
+/// Executes a test case of the test program.
+///
+/// This method is intended to be called within a subprocess and is expected
+/// to terminate execution either by exec(2)ing the test program or by
+/// exiting with a failure.
+///
+/// \param test_program The test program to execute.
+/// \param test_case_name Name of the test case to invoke.
+/// \param vars User-provided variables to pass to the test program.
+void
+engine::tap_interface::exec_test(
+ const model::test_program& test_program,
+ const std::string& test_case_name,
+ const config::properties_map& vars,
+ const fs::path& /* control_directory */) const
+{
+ PRE(test_case_name == "main");
+
+ for (config::properties_map::const_iterator iter = vars.begin();
+ iter != vars.end(); ++iter) {
+ utils::setenv(F("TEST_ENV_%s") % (*iter).first, (*iter).second);
+ }
+
+ process::args_vector args;
+ process::exec(test_program.absolute_path(), args);
+}
+
+
+/// Computes the result of a test case based on its termination status.
+///
+/// \param status The termination status of the subprocess used to execute
+/// the exec_test() method or none if the test timed out.
+/// \param stdout_path Path to the file containing the stdout of the test.
+///
+/// \return A test result.
+model::test_result
+engine::tap_interface::compute_result(
+ const optional< process::status >& status,
+ const fs::path& /* control_directory */,
+ const fs::path& stdout_path,
+ const fs::path& /* stderr_path */) const
+{
+ if (!status) {
+ return model::test_result(model::test_result_broken,
+ "Test case timed out");
+ } else {
+ if (status.get().signaled()) {
+ return model::test_result(
+ model::test_result_broken,
+ F("Received signal %s") % status.get().termsig());
+ } else {
+ try {
+ const tap_summary summary = parse_tap_output(stdout_path);
+ return tap_to_result(summary, status.get());
+ } catch (const load_error& e) {
+ return model::test_result(
+ model::test_result_broken,
+ F("TAP test program yielded invalid data: %s") % e.what());
+ }
+ }
+ }
+}
diff --git a/engine/tap.hpp b/engine/tap.hpp
new file mode 100644
index 000000000000..b46bf28f0240
--- /dev/null
+++ b/engine/tap.hpp
@@ -0,0 +1,67 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/tap.hpp
+/// Execution engine for test programs that output the TAP protocol.
+
+#if !defined(ENGINE_TAP_HPP)
+#define ENGINE_TAP_HPP
+
+#include "engine/scheduler.hpp"
+
+namespace engine {
+
+
+/// Implementation of the scheduler interface for tap test programs.
+class tap_interface : public engine::scheduler::interface {
+public:
+ void exec_list(const model::test_program&,
+ const utils::config::properties_map&) const UTILS_NORETURN;
+
+ model::test_cases_map parse_list(
+ const utils::optional< utils::process::status >&,
+ const utils::fs::path&,
+ const utils::fs::path&) const;
+
+ void exec_test(const model::test_program&, const std::string&,
+ const utils::config::properties_map&,
+ const utils::fs::path&) const
+ UTILS_NORETURN;
+
+ model::test_result compute_result(
+ const utils::optional< utils::process::status >&,
+ const utils::fs::path&,
+ const utils::fs::path&,
+ const utils::fs::path&) const;
+};
+
+
+} // namespace engine
+
+
+#endif // !defined(ENGINE_TAP_HPP)
diff --git a/engine/tap_helpers.cpp b/engine/tap_helpers.cpp
new file mode 100644
index 000000000000..4f9505c78dec
--- /dev/null
+++ b/engine/tap_helpers.cpp
@@ -0,0 +1,202 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+extern "C" {
+#include <sys/stat.h>
+
+#include <unistd.h>
+
+extern char** environ;
+}
+
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+
+#include "utils/env.hpp"
+#include "utils/format/containers.ipp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/optional.ipp"
+#include "utils/test_utils.ipp"
+
+namespace fs = utils::fs;
+
+
+namespace {
+
+
+/// Logs an error message and exits the test with an error code.
+///
+/// \param str The error message to log.
+static void
+fail(const std::string& str)
+{
+ std::cerr << str << '\n';
+ std::exit(EXIT_FAILURE);
+}
+
+
+/// A test scenario that validates the TEST_ENV_* variables.
+static void
+test_check_configuration_variables(void)
+{
+ std::set< std::string > vars;
+ char** iter;
+ for (iter = environ; *iter != NULL; ++iter) {
+ if (std::strstr(*iter, "TEST_ENV_") == *iter) {
+ vars.insert(*iter);
+ }
+ }
+
+ std::set< std::string > exp_vars;
+ exp_vars.insert("TEST_ENV_first=some value");
+ exp_vars.insert("TEST_ENV_second=some other value");
+ if (vars == exp_vars) {
+ std::cout << "1..1\n"
+ << "ok 1\n";
+ } else {
+ std::cout << "1..1\n"
+ << "not ok 1\n"
+ << F(" Expected: %s\nFound: %s\n") % exp_vars % vars;
+ }
+}
+
+
+/// A test scenario that crashes.
+static void
+test_crash(void)
+{
+ utils::abort_without_coredump();
+}
+
+
+/// A test scenario that reports some tests as failed.
+static void
+test_fail(void)
+{
+ std::cout << "1..5\n"
+ << "ok 1 - This is good!\n"
+ << "not ok 2\n"
+ << "ok 3 - TODO Consider this as passed\n"
+ << "ok 4\n"
+ << "not ok 5\n";
+}
+
+
+/// A test scenario that passes.
+static void
+test_pass(void)
+{
+ std::cout << "1..4\n"
+ << "ok 1 - This is good!\n"
+ << "non-result data\n"
+ << "ok 2 - SKIP Consider this as passed\n"
+ << "ok 3 - TODO Consider this as passed\n"
+ << "ok 4\n";
+}
+
+
+/// A test scenario that passes but then exits with non-zero.
+static void
+test_pass_but_exit_failure(void)
+{
+ std::cout << "1..2\n"
+ << "ok 1\n"
+ << "ok 2\n";
+ std::exit(70);
+}
+
+
+/// A test scenario that times out.
+///
+/// Note that the timeout is defined in the Kyuafile, as the TAP interface has
+/// no means for test programs to specify this by themselves.
+static void
+test_timeout(void)
+{
+ std::cout << "1..2\n"
+ << "ok 1\n";
+
+ ::sleep(10);
+ const fs::path control_dir = fs::path(utils::getenv("CONTROL_DIR").get());
+ std::ofstream file((control_dir / "cookie").c_str());
+ if (!file)
+ fail("Failed to create the control cookie");
+ file.close();
+}
+
+
+} // anonymous namespace
+
+
+/// Entry point to the test program.
+///
+/// The caller can select which test scenario to run by modifying the program's
+/// basename on disk (either by a copy or by a hard link).
+///
+/// \todo It may be worth to split this binary into separate, smaller binaries,
+/// one for every "test scenario". We use this program as a dispatcher for
+/// different "main"s, the only reason being to keep the amount of helper test
+/// programs to a minimum. However, putting this each function in its own
+/// binary could simplify many other things.
+///
+/// \param argc The number of CLI arguments.
+/// \param argv The CLI arguments themselves. These are not used because
+/// Kyua will not pass any arguments to the plain test program.
+int
+main(int argc, char** argv)
+{
+ if (argc != 1) {
+ std::cerr << "No arguments allowed; select the test scenario with the "
+ "program's basename\n";
+ return EXIT_FAILURE;
+ }
+
+ const std::string& test_scenario = fs::path(argv[0]).leaf_name();
+
+ if (test_scenario == "check_configuration_variables")
+ test_check_configuration_variables();
+ else if (test_scenario == "crash")
+ test_crash();
+ else if (test_scenario == "fail")
+ test_fail();
+ else if (test_scenario == "pass")
+ test_pass();
+ else if (test_scenario == "pass_but_exit_failure")
+ test_pass_but_exit_failure();
+ else if (test_scenario == "timeout")
+ test_timeout();
+ else {
+ std::cerr << "Unknown test scenario\n";
+ return EXIT_FAILURE;
+ }
+
+ return EXIT_SUCCESS;
+}
diff --git a/engine/tap_parser.cpp b/engine/tap_parser.cpp
new file mode 100644
index 000000000000..d41328534fad
--- /dev/null
+++ b/engine/tap_parser.cpp
@@ -0,0 +1,438 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/tap_parser.hpp"
+
+#include <fstream>
+
+#include "engine/exceptions.hpp"
+#include "utils/format/macros.hpp"
+#include "utils/noncopyable.hpp"
+#include "utils/optional.ipp"
+#include "utils/sanity.hpp"
+#include "utils/text/exceptions.hpp"
+#include "utils/text/operations.ipp"
+#include "utils/text/regex.hpp"
+
+namespace fs = utils::fs;
+namespace text = utils::text;
+
+using utils::optional;
+
+
+/// TAP plan representing all tests being skipped.
+const engine::tap_plan engine::all_skipped_plan(1, 0);
+
+
+namespace {
+
+
+/// Implementation of the TAP parser.
+///
+/// This is a class only to simplify keeping global constant values around (like
+/// prebuilt regular expressions).
+class tap_parser : utils::noncopyable {
+ /// Regular expression to match plan lines.
+ text::regex _plan_regex;
+
+ /// Regular expression to match a TODO and extract the reason.
+ text::regex _todo_regex;
+
+ /// Regular expression to match a SKIP and extract the reason.
+ text::regex _skip_regex;
+
+ /// Regular expression to match a single test result.
+ text::regex _result_regex;
+
+ /// Checks if a line contains a TAP plan and extracts its data.
+ ///
+ /// \param line The line to try to parse.
+ /// \param [in,out] out_plan Used to store the found plan, if any. The same
+ /// output variable should be given to all calls to this function so
+ /// that duplicate plan entries can be discovered.
+ /// \param [out] out_all_skipped_reason Used to store the reason for all
+ /// tests being skipped, if any. If this is set to a non-empty value,
+ /// then the out_plan is set to 1..0.
+ ///
+ /// \return True if the line matched a plan; false otherwise.
+ ///
+ /// \throw engine::format_error If the input is invalid.
+ /// \throw text::error If the input is invalid.
+ bool
+ try_parse_plan(const std::string& line,
+ optional< engine::tap_plan >& out_plan,
+ std::string& out_all_skipped_reason)
+ {
+ const text::regex_matches plan_matches = _plan_regex.match(line);
+ if (!plan_matches)
+ return false;
+ const engine::tap_plan plan(
+ text::to_type< std::size_t >(plan_matches.get(1)),
+ text::to_type< std::size_t >(plan_matches.get(2)));
+
+ if (out_plan)
+ throw engine::format_error(
+ F("Found duplicate plan %s..%s (saw %s..%s earlier)") %
+ plan.first % plan.second %
+ out_plan.get().first % out_plan.get().second);
+
+ std::string all_skipped_reason;
+ const text::regex_matches skip_matches = _skip_regex.match(line);
+ if (skip_matches) {
+ if (plan != engine::all_skipped_plan) {
+ throw engine::format_error(F("Skipped plan must be %s..%s") %
+ engine::all_skipped_plan.first %
+ engine::all_skipped_plan.second);
+ }
+ all_skipped_reason = skip_matches.get(2);
+ if (all_skipped_reason.empty())
+ all_skipped_reason = "No reason specified";
+ } else {
+ if (plan.first > plan.second)
+ throw engine::format_error(F("Found reversed plan %s..%s") %
+ plan.first % plan.second);
+ }
+
+ INV(!out_plan);
+ out_plan = plan;
+ out_all_skipped_reason = all_skipped_reason;
+
+ POST(out_plan);
+ POST(out_all_skipped_reason.empty() ||
+ out_plan.get() == engine::all_skipped_plan);
+
+ return true;
+ }
+
+ /// Checks if a line contains a TAP test result and extracts its data.
+ ///
+ /// \param line The line to try to parse.
+ /// \param [in,out] out_ok_count Accumulator for 'ok' results.
+ /// \param [in,out] out_not_ok_count Accumulator for 'not ok' results.
+ /// \param [out] out_bailed_out Set to true if the test bailed out.
+ ///
+ /// \return True if the line matched a result; false otherwise.
+ ///
+ /// \throw engine::format_error If the input is invalid.
+ /// \throw text::error If the input is invalid.
+ bool
+ try_parse_result(const std::string& line, std::size_t& out_ok_count,
+ std::size_t& out_not_ok_count, bool& out_bailed_out)
+ {
+ PRE(!out_bailed_out);
+
+ const text::regex_matches result_matches = _result_regex.match(line);
+ if (result_matches) {
+ if (result_matches.get(1) == "ok") {
+ ++out_ok_count;
+ } else {
+ INV(result_matches.get(1) == "not ok");
+ if (_todo_regex.match(line) || _skip_regex.match(line)) {
+ ++out_ok_count;
+ } else {
+ ++out_not_ok_count;
+ }
+ }
+ return true;
+ } else {
+ if (line.find("Bail out!") == 0) {
+ out_bailed_out = true;
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+public:
+ /// Sets up the TAP parser state.
+ tap_parser(void) :
+ _plan_regex(text::regex::compile("^([0-9]+)\\.\\.([0-9]+)", 2)),
+ _todo_regex(text::regex::compile("TODO[ \t]*(.*)$", 2, true)),
+ _skip_regex(text::regex::compile("(SKIP|Skipped:?)[ \t]*(.*)$", 2,
+ true)),
+ _result_regex(text::regex::compile("^(not ok|ok)[ \t-]+[0-9]*", 1))
+ {
+ }
+
+ /// Parses an input file containing TAP output.
+ ///
+ /// \param input The stream to read from.
+ ///
+ /// \return The results of the parsing in the form of a tap_summary object.
+ ///
+ /// \throw engine::format_error If there are any syntax errors in the input.
+ /// \throw text::error If there are any syntax errors in the input.
+ engine::tap_summary
+ parse(std::ifstream& input)
+ {
+ optional< engine::tap_plan > plan;
+ std::string all_skipped_reason;
+ bool bailed_out = false;
+ std::size_t ok_count = 0, not_ok_count = 0;
+
+ std::string line;
+ while (!bailed_out && std::getline(input, line)) {
+ if (try_parse_result(line, ok_count, not_ok_count, bailed_out))
+ continue;
+ (void)try_parse_plan(line, plan, all_skipped_reason);
+ }
+
+ if (bailed_out) {
+ return engine::tap_summary::new_bailed_out();
+ } else {
+ if (!plan)
+ throw engine::format_error(
+ "Output did not contain any TAP plan and the program did "
+ "not bail out");
+
+ if (plan.get() == engine::all_skipped_plan) {
+ return engine::tap_summary::new_all_skipped(all_skipped_reason);
+ } else {
+ const std::size_t exp_count = plan.get().second -
+ plan.get().first + 1;
+ const std::size_t actual_count = ok_count + not_ok_count;
+ if (exp_count != actual_count) {
+ throw engine::format_error(
+ "Reported plan differs from actual executed tests");
+ }
+ return engine::tap_summary::new_results(plan.get(), ok_count,
+ not_ok_count);
+ }
+ }
+ }
+};
+
+
+} // anonymous namespace
+
+
+/// Constructs a TAP summary with the results of parsing a TAP output.
+///
+/// \param bailed_out_ Whether the test program bailed out early or not.
+/// \param plan_ The TAP plan.
+/// \param all_skipped_reason_ The reason for skipping all tests, if any.
+/// \param ok_count_ Number of 'ok' test results.
+/// \param not_ok_count_ Number of 'not ok' test results.
+engine::tap_summary::tap_summary(const bool bailed_out_,
+ const tap_plan& plan_,
+ const std::string& all_skipped_reason_,
+ const std::size_t ok_count_,
+ const std::size_t not_ok_count_) :
+ _bailed_out(bailed_out_), _plan(plan_),
+ _all_skipped_reason(all_skipped_reason_),
+ _ok_count(ok_count_), _not_ok_count(not_ok_count_)
+{
+}
+
+
+/// Constructs a TAP summary for a bailed out test program.
+///
+/// \return The new tap_summary object.
+engine::tap_summary
+engine::tap_summary::new_bailed_out(void)
+{
+ return tap_summary(true, tap_plan(0, 0), "", 0, 0);
+}
+
+
+/// Constructs a TAP summary for a test program that skipped all tests.
+///
+/// \param reason Textual reason describing why the tests were skipped.
+///
+/// \return The new tap_summary object.
+engine::tap_summary
+engine::tap_summary::new_all_skipped(const std::string& reason)
+{
+ return tap_summary(false, all_skipped_plan, reason, 0, 0);
+}
+
+
+/// Constructs a TAP summary for a test program that reported results.
+///
+/// \param plan_ The TAP plan.
+/// \param ok_count_ Total number of 'ok' results.
+/// \param not_ok_count_ Total number of 'not ok' results.
+///
+/// \return The new tap_summary object.
+engine::tap_summary
+engine::tap_summary::new_results(const tap_plan& plan_,
+ const std::size_t ok_count_,
+ const std::size_t not_ok_count_)
+{
+ PRE((plan_.second - plan_.first + 1) == (ok_count_ + not_ok_count_));
+ return tap_summary(false, plan_, "", ok_count_, not_ok_count_);
+}
+
+
+/// Checks whether the test program bailed out early or not.
+///
+/// \return True if the test program aborted execution before completing.
+bool
+engine::tap_summary::bailed_out(void) const
+{
+ return _bailed_out;
+}
+
+
+/// Gets the TAP plan of the test program.
+///
+/// \pre bailed_out() must be false.
+///
+/// \return The TAP plan. If 1..0, then all_skipped_reason() will have some
+/// contents.
+const engine::tap_plan&
+engine::tap_summary::plan(void) const
+{
+ PRE(!_bailed_out);
+ return _plan;
+}
+
+
+/// Gets the reason for skipping all the tests, if any.
+///
+/// \pre bailed_out() must be false.
+/// \pre plan() returns 1..0.
+///
+/// \return The reason for skipping all the tests.
+const std::string&
+engine::tap_summary::all_skipped_reason(void) const
+{
+ PRE(!_bailed_out);
+ PRE(_plan == all_skipped_plan);
+ return _all_skipped_reason;
+}
+
+
+/// Gets the number of 'ok' test results.
+///
+/// \pre bailed_out() must be false.
+///
+/// \return The number of test results that reported 'ok'.
+std::size_t
+engine::tap_summary::ok_count(void) const
+{
+ PRE(!bailed_out());
+ PRE(_all_skipped_reason.empty());
+ return _ok_count;
+}
+
+
+/// Gets the number of 'not ok' test results.
+///
+/// \pre bailed_out() must be false.
+///
+/// \return The number of test results that reported 'not ok'.
+std::size_t
+engine::tap_summary::not_ok_count(void) const
+{
+ PRE(!_bailed_out);
+ PRE(_all_skipped_reason.empty());
+ return _not_ok_count;
+}
+
+
+/// Checks two tap_summary objects for equality.
+///
+/// \param other The object to compare this one to.
+///
+/// \return True if the two objects are equal; false otherwise.
+bool
+engine::tap_summary::operator==(const tap_summary& other) const
+{
+ return (_bailed_out == other._bailed_out &&
+ _plan == other._plan &&
+ _all_skipped_reason == other._all_skipped_reason &&
+ _ok_count == other._ok_count &&
+ _not_ok_count == other._not_ok_count);
+}
+
+
+/// Checks two tap_summary objects for inequality.
+///
+/// \param other The object to compare this one to.
+///
+/// \return True if the two objects are different; false otherwise.
+bool
+engine::tap_summary::operator!=(const tap_summary& other) const
+{
+ return !(*this == other);
+}
+
+
+/// Formats a tap_summary into a stream.
+///
+/// \param output The stream into which to inject the object.
+/// \param summary The summary to format.
+///
+/// \return The output stream.
+std::ostream&
+engine::operator<<(std::ostream& output, const tap_summary& summary)
+{
+ output << "tap_summary{";
+ if (summary.bailed_out()) {
+ output << "bailed_out=true";
+ } else {
+ const tap_plan& plan = summary.plan();
+ output << "bailed_out=false"
+ << ", plan=" << plan.first << ".." << plan.second;
+ if (plan == all_skipped_plan) {
+ output << ", all_skipped_reason=" << summary.all_skipped_reason();
+ } else {
+ output << ", ok_count=" << summary.ok_count()
+ << ", not_ok_count=" << summary.not_ok_count();
+ }
+ }
+ output << "}";
+ return output;
+}
+
+
+/// Parses an input file containing the TAP output of a test program.
+///
+/// \param filename Path to the file to parse.
+///
+/// \return The parsed data in the form of a tap_summary.
+///
+/// \throw load_error If there are any problems parsing the file. Such problems
+/// should be considered as test program breakage.
+engine::tap_summary
+engine::parse_tap_output(const utils::fs::path& filename)
+{
+ std::ifstream input(filename.str().c_str());
+ if (!input)
+ throw engine::load_error(filename, "Failed to open TAP output file");
+
+ try {
+ return tap_summary(tap_parser().parse(input));
+ } catch (const engine::format_error& e) {
+ throw engine::load_error(filename, e.what());
+ } catch (const text::error& e) {
+ throw engine::load_error(filename, e.what());
+ }
+}
diff --git a/engine/tap_parser.hpp b/engine/tap_parser.hpp
new file mode 100644
index 000000000000..84cea908f5ba
--- /dev/null
+++ b/engine/tap_parser.hpp
@@ -0,0 +1,99 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/tap_parser.hpp
+/// Utilities to parse TAP test program output.
+
+#if !defined(ENGINE_TAP_PARSER_HPP)
+#define ENGINE_TAP_PARSER_HPP
+
+#include "engine/tap_parser_fwd.hpp"
+
+#include <cstddef>
+#include <ostream>
+#include <string>
+
+#include "utils/fs/path_fwd.hpp"
+
+namespace engine {
+
+
+/// TAP plan representing all tests being skipped.
+extern const engine::tap_plan all_skipped_plan;
+
+
+/// TAP output representation and parser.
+class tap_summary {
+ /// Whether the test program bailed out early or not.
+ bool _bailed_out;
+
+ /// The TAP plan. Only valid if not bailed out.
+ tap_plan _plan;
+
+ /// If not empty, the reason why all tests were skipped.
+ std::string _all_skipped_reason;
+
+ /// Total number of 'ok' tests. Only valid if not balied out.
+ std::size_t _ok_count;
+
+ /// Total number of 'not ok' tests. Only valid if not balied out.
+ std::size_t _not_ok_count;
+
+ tap_summary(const bool, const tap_plan&, const std::string&,
+ const std::size_t, const std::size_t);
+
+public:
+ // Yes, these three constructors indicate that we really ought to have three
+ // different classes and select between them at runtime. But doing so would
+ // be overly complex for our really simple needs here.
+ static tap_summary new_bailed_out(void);
+ static tap_summary new_all_skipped(const std::string&);
+ static tap_summary new_results(const tap_plan&, const std::size_t,
+ const std::size_t);
+
+ bool bailed_out(void) const;
+ const tap_plan& plan(void) const;
+ const std::string& all_skipped_reason(void) const;
+ std::size_t ok_count(void) const;
+ std::size_t not_ok_count(void) const;
+
+ bool operator==(const tap_summary&) const;
+ bool operator!=(const tap_summary&) const;
+};
+
+
+std::ostream& operator<<(std::ostream&, const tap_summary&);
+
+
+tap_summary parse_tap_output(const utils::fs::path&);
+
+
+} // namespace engine
+
+
+#endif // !defined(ENGINE_TAP_PARSER_HPP)
diff --git a/engine/tap_parser_fwd.hpp b/engine/tap_parser_fwd.hpp
new file mode 100644
index 000000000000..481ed2f42267
--- /dev/null
+++ b/engine/tap_parser_fwd.hpp
@@ -0,0 +1,50 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/// \file engine/tap_parser_fwd.hpp
+/// Forward declarations for engine/tap_parser.hpp
+
+#if !defined(ENGINE_TAP_PARSER_FWD_HPP)
+#define ENGINE_TAP_PARSER_FWD_HPP
+
+#include <cstddef>
+#include <utility>
+
+namespace engine {
+
+
+/// Representation of the TAP plan line.
+typedef std::pair< std::size_t, std::size_t > tap_plan;
+
+
+class tap_summary;
+
+
+} // namespace engine
+
+#endif // !defined(ENGINE_TAP_PARSER_FWD_HPP)
diff --git a/engine/tap_parser_test.cpp b/engine/tap_parser_test.cpp
new file mode 100644
index 000000000000..af993bfab4ab
--- /dev/null
+++ b/engine/tap_parser_test.cpp
@@ -0,0 +1,465 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/tap_parser.hpp"
+
+#include <fstream>
+
+#include <atf-c++.hpp>
+
+#include "engine/exceptions.hpp"
+#include "utils/format/containers.ipp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/path.hpp"
+
+namespace fs = utils::fs;
+
+
+namespace {
+
+
+/// Helper to execute parse_tap_output() on inline text contents.
+///
+/// \param contents The TAP output to parse.
+///
+/// \return The tap_summary object resultingafter the parse.
+///
+/// \throw engine::load_error If parse_tap_output() fails.
+static engine::tap_summary
+do_parse(const std::string& contents)
+{
+ std::ofstream output("tap.txt");
+ ATF_REQUIRE(output);
+ output << contents;
+ output.close();
+ return engine::parse_tap_output(fs::path("tap.txt"));
+}
+
+
+} // anonymous namespace
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__bailed_out);
+ATF_TEST_CASE_BODY(tap_summary__bailed_out)
+{
+ const engine::tap_summary summary = engine::tap_summary::new_bailed_out();
+ ATF_REQUIRE(summary.bailed_out());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__some_results);
+ATF_TEST_CASE_BODY(tap_summary__some_results)
+{
+ const engine::tap_summary summary = engine::tap_summary::new_results(
+ engine::tap_plan(1, 5), 3, 2);
+ ATF_REQUIRE(!summary.bailed_out());
+ ATF_REQUIRE_EQ(engine::tap_plan(1, 5), summary.plan());
+ ATF_REQUIRE_EQ(3, summary.ok_count());
+ ATF_REQUIRE_EQ(2, summary.not_ok_count());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__all_skipped);
+ATF_TEST_CASE_BODY(tap_summary__all_skipped)
+{
+ const engine::tap_summary summary = engine::tap_summary::new_all_skipped(
+ "Skipped");
+ ATF_REQUIRE(!summary.bailed_out());
+ ATF_REQUIRE_EQ(engine::tap_plan(1, 0), summary.plan());
+ ATF_REQUIRE_EQ("Skipped", summary.all_skipped_reason());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__equality_operators);
+ATF_TEST_CASE_BODY(tap_summary__equality_operators)
+{
+ const engine::tap_summary bailed_out =
+ engine::tap_summary::new_bailed_out();
+ const engine::tap_summary all_skipped_1 =
+ engine::tap_summary::new_all_skipped("Reason 1");
+ const engine::tap_summary results_1 =
+ engine::tap_summary::new_results(engine::tap_plan(1, 5), 3, 2);
+
+ // Self-equality checks.
+ ATF_REQUIRE( bailed_out == bailed_out);
+ ATF_REQUIRE(!(bailed_out != bailed_out));
+ ATF_REQUIRE( all_skipped_1 == all_skipped_1);
+ ATF_REQUIRE(!(all_skipped_1 != all_skipped_1));
+ ATF_REQUIRE( results_1 == results_1);
+ ATF_REQUIRE(!(results_1 != results_1));
+
+ // Cross-equality checks.
+ ATF_REQUIRE(!(bailed_out == all_skipped_1));
+ ATF_REQUIRE( bailed_out != all_skipped_1);
+ ATF_REQUIRE(!(bailed_out == results_1));
+ ATF_REQUIRE( bailed_out != results_1);
+ ATF_REQUIRE(!(all_skipped_1 == results_1));
+ ATF_REQUIRE( all_skipped_1 != results_1);
+
+ // Checks for the all_skipped "type".
+ const engine::tap_summary all_skipped_2 =
+ engine::tap_summary::new_all_skipped("Reason 2");
+ ATF_REQUIRE(!(all_skipped_1 == all_skipped_2));
+ ATF_REQUIRE( all_skipped_1 != all_skipped_2);
+
+
+ // Checks for the results "type", different plan.
+ const engine::tap_summary results_2 =
+ engine::tap_summary::new_results(engine::tap_plan(2, 6),
+ results_1.ok_count(),
+ results_1.not_ok_count());
+ ATF_REQUIRE(!(results_1 == results_2));
+ ATF_REQUIRE( results_1 != results_2);
+
+
+ // Checks for the results "type", different counts.
+ const engine::tap_summary results_3 =
+ engine::tap_summary::new_results(results_1.plan(),
+ results_1.not_ok_count(),
+ results_1.ok_count());
+ ATF_REQUIRE(!(results_1 == results_3));
+ ATF_REQUIRE( results_1 != results_3);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__output);
+ATF_TEST_CASE_BODY(tap_summary__output)
+{
+ {
+ const engine::tap_summary summary =
+ engine::tap_summary::new_bailed_out();
+ ATF_REQUIRE_EQ(
+ "tap_summary{bailed_out=true}",
+ (F("%s") % summary).str());
+ }
+
+ {
+ const engine::tap_summary summary =
+ engine::tap_summary::new_results(engine::tap_plan(5, 10), 2, 4);
+ ATF_REQUIRE_EQ(
+ "tap_summary{bailed_out=false, plan=5..10, ok_count=2, "
+ "not_ok_count=4}",
+ (F("%s") % summary).str());
+ }
+
+ {
+ const engine::tap_summary summary =
+ engine::tap_summary::new_all_skipped("Who knows");
+ ATF_REQUIRE_EQ(
+ "tap_summary{bailed_out=false, plan=1..0, "
+ "all_skipped_reason=Who knows}",
+ (F("%s") % summary).str());
+ }
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__only_one_result);
+ATF_TEST_CASE_BODY(parse_tap_output__only_one_result)
+{
+ const engine::tap_summary summary = do_parse(
+ "1..1\n"
+ "ok - 1\n");
+
+ const engine::tap_summary exp_summary =
+ engine::tap_summary::new_results(engine::tap_plan(1, 1), 1, 0);
+ ATF_REQUIRE_EQ(exp_summary, summary);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__all_pass);
+ATF_TEST_CASE_BODY(parse_tap_output__all_pass)
+{
+ const engine::tap_summary summary = do_parse(
+ "1..8\n"
+ "ok - 1\n"
+ " Some diagnostic message\n"
+ "ok - 2 This test also passed\n"
+ "garbage line\n"
+ "ok - 3 This test passed\n"
+ "not ok 4 # SKIP Some reason\n"
+ "not ok 5 # TODO Another reason\n"
+ "ok - 6 Doesn't make a difference SKIP\n"
+ "ok - 7 Doesn't make a difference either TODO\n"
+ "ok # Also works without a number\n");
+
+ const engine::tap_summary exp_summary =
+ engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0);
+ ATF_REQUIRE_EQ(exp_summary, summary);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__some_fail);
+ATF_TEST_CASE_BODY(parse_tap_output__some_fail)
+{
+ const engine::tap_summary summary = do_parse(
+ "garbage line\n"
+ "not ok - 1 This test failed\n"
+ "ok - 2 This test passed\n"
+ "not ok - 3 This test failed\n"
+ "1..6\n"
+ "not ok - 4 This test failed\n"
+ "ok - 5 This test passed\n"
+ "not ok # Fails as well without a number\n");
+
+ const engine::tap_summary exp_summary =
+ engine::tap_summary::new_results(engine::tap_plan(1, 6), 2, 4);
+ ATF_REQUIRE_EQ(exp_summary, summary);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_and_todo_variants);
+ATF_TEST_CASE_BODY(parse_tap_output__skip_and_todo_variants)
+{
+ const engine::tap_summary summary = do_parse(
+ "1..8\n"
+ "not ok - 1 # SKIP Some reason\n"
+ "not ok - 2 # skip Some reason\n"
+ "not ok - 3 # Skipped Some reason\n"
+ "not ok - 4 # skipped Some reason\n"
+ "not ok - 5 # Skipped: Some reason\n"
+ "not ok - 6 # skipped: Some reason\n"
+ "not ok - 7 # TODO Some reason\n"
+ "not ok - 8 # todo Some reason\n");
+
+ const engine::tap_summary exp_summary =
+ engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0);
+ ATF_REQUIRE_EQ(exp_summary, summary);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_with_reason);
+ATF_TEST_CASE_BODY(parse_tap_output__skip_all_with_reason)
+{
+ const engine::tap_summary summary = do_parse(
+ "1..0 SKIP Some reason for skipping\n"
+ "ok - 1\n"
+ " Some diagnostic message\n"
+ "ok - 6 Doesn't make a difference SKIP\n"
+ "ok - 7 Doesn't make a difference either TODO\n");
+
+ const engine::tap_summary exp_summary =
+ engine::tap_summary::new_all_skipped("Some reason for skipping");
+ ATF_REQUIRE_EQ(exp_summary, summary);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_without_reason);
+ATF_TEST_CASE_BODY(parse_tap_output__skip_all_without_reason)
+{
+ const engine::tap_summary summary = do_parse(
+ "1..0 unrecognized # garbage skip\n");
+
+ const engine::tap_summary exp_summary =
+ engine::tap_summary::new_all_skipped("No reason specified");
+ ATF_REQUIRE_EQ(exp_summary, summary);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_invalid);
+ATF_TEST_CASE_BODY(parse_tap_output__skip_all_invalid)
+{
+ ATF_REQUIRE_THROW_RE(engine::load_error,
+ "Skipped plan must be 1\\.\\.0",
+ do_parse("1..3 # skip\n"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__plan_at_end);
+ATF_TEST_CASE_BODY(parse_tap_output__plan_at_end)
+{
+ const engine::tap_summary summary = do_parse(
+ "ok - 1\n"
+ " Some diagnostic message\n"
+ "ok - 2 This test also passed\n"
+ "garbage line\n"
+ "ok - 3 This test passed\n"
+ "not ok 4 # SKIP Some reason\n"
+ "not ok 5 # TODO Another reason\n"
+ "ok - 6 Doesn't make a difference SKIP\n"
+ "ok - 7 Doesn't make a difference either TODO\n"
+ "1..7\n");
+
+ const engine::tap_summary exp_summary =
+ engine::tap_summary::new_results(engine::tap_plan(1, 7), 7, 0);
+ ATF_REQUIRE_EQ(exp_summary, summary);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__stray_oks);
+ATF_TEST_CASE_BODY(parse_tap_output__stray_oks)
+{
+ const engine::tap_summary summary = do_parse(
+ "1..3\n"
+ "ok - 1\n"
+ "ok\n"
+ "ok - 2 This test also passed\n"
+ "not ok\n"
+ "ok - 3 This test passed\n");
+
+ const engine::tap_summary exp_summary =
+ engine::tap_summary::new_results(engine::tap_plan(1, 3), 3, 0);
+ ATF_REQUIRE_EQ(exp_summary, summary);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__no_plan);
+ATF_TEST_CASE_BODY(parse_tap_output__no_plan)
+{
+ ATF_REQUIRE_THROW_RE(
+ engine::load_error,
+ "Output did not contain any TAP plan",
+ do_parse(
+ "not ok - 1 This test failed\n"
+ "ok - 2 This test passed\n"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__double_plan);
+ATF_TEST_CASE_BODY(parse_tap_output__double_plan)
+{
+ ATF_REQUIRE_THROW_RE(
+ engine::load_error,
+ "Found duplicate plan",
+ do_parse(
+ "garbage line\n"
+ "1..5\n"
+ "not ok - 1 This test failed\n"
+ "ok - 2 This test passed\n"
+ "1..8\n"
+ "ok\n"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_plan);
+ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_plan)
+{
+ ATF_REQUIRE_THROW_RE(
+ engine::load_error,
+ "Reported plan differs from actual executed tests",
+ do_parse(
+ "1..3\n"
+ "not ok - 1 This test failed\n"
+ "ok - 2 This test passed\n"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_trailing_plan);
+ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_trailing_plan)
+{
+ ATF_REQUIRE_THROW_RE(
+ engine::load_error,
+ "Reported plan differs from actual executed tests",
+ do_parse(
+ "not ok - 1 This test failed\n"
+ "ok - 2 This test passed\n"
+ "1..3\n"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__insane_plan);
+ATF_TEST_CASE_BODY(parse_tap_output__insane_plan)
+{
+ ATF_REQUIRE_THROW_RE(
+ engine::load_error, "Invalid value",
+ do_parse("120830981209831..234891793874080981092803981092312\n"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__reversed_plan);
+ATF_TEST_CASE_BODY(parse_tap_output__reversed_plan)
+{
+ ATF_REQUIRE_THROW_RE(engine::load_error,
+ "Found reversed plan 8\\.\\.5",
+ do_parse("8..5\n"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out);
+ATF_TEST_CASE_BODY(parse_tap_output__bail_out)
+{
+ const engine::tap_summary summary = do_parse(
+ "1..3\n"
+ "not ok - 1 This test failed\n"
+ "Bail out! There is some unknown problem\n"
+ "ok - 2 This test passed\n");
+
+ const engine::tap_summary exp_summary =
+ engine::tap_summary::new_bailed_out();
+ ATF_REQUIRE_EQ(exp_summary, summary);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out_wins_over_no_plan);
+ATF_TEST_CASE_BODY(parse_tap_output__bail_out_wins_over_no_plan)
+{
+ const engine::tap_summary summary = do_parse(
+ "not ok - 1 This test failed\n"
+ "Bail out! There is some unknown problem\n"
+ "ok - 2 This test passed\n");
+
+ const engine::tap_summary exp_summary =
+ engine::tap_summary::new_bailed_out();
+ ATF_REQUIRE_EQ(exp_summary, summary);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__open_failure);
+ATF_TEST_CASE_BODY(parse_tap_output__open_failure)
+{
+ ATF_REQUIRE_THROW_RE(engine::load_error, "hello.txt.*Failed to open",
+ engine::parse_tap_output(fs::path("hello.txt")));
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ ATF_ADD_TEST_CASE(tcs, tap_summary__bailed_out);
+ ATF_ADD_TEST_CASE(tcs, tap_summary__some_results);
+ ATF_ADD_TEST_CASE(tcs, tap_summary__all_skipped);
+ ATF_ADD_TEST_CASE(tcs, tap_summary__equality_operators);
+ ATF_ADD_TEST_CASE(tcs, tap_summary__output);
+
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__only_one_result);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__all_pass);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__some_fail);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_and_todo_variants);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_without_reason);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_with_reason);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_invalid);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__plan_at_end);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__stray_oks);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__no_plan);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__double_plan);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_plan);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_trailing_plan);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__insane_plan);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__reversed_plan);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out_wins_over_no_plan);
+ ATF_ADD_TEST_CASE(tcs, parse_tap_output__open_failure);
+}
diff --git a/engine/tap_test.cpp b/engine/tap_test.cpp
new file mode 100644
index 000000000000..f4253a68e727
--- /dev/null
+++ b/engine/tap_test.cpp
@@ -0,0 +1,218 @@
+// Copyright 2015 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/tap.hpp"
+
+extern "C" {
+#include <signal.h>
+}
+
+#include <atf-c++.hpp>
+
+#include "engine/config.hpp"
+#include "engine/scheduler.hpp"
+#include "model/metadata.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "model/test_result.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/datetime.hpp"
+#include "utils/env.hpp"
+#include "utils/format/containers.ipp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/optional.ipp"
+
+namespace config = utils::config;
+namespace datetime = utils::datetime;
+namespace fs = utils::fs;
+namespace scheduler = engine::scheduler;
+
+using utils::none;
+
+
+namespace {
+
+
+/// Copies the tap helper to the work directory, selecting a specific helper.
+///
+/// \param tc Pointer to the calling test case, to obtain srcdir.
+/// \param name Name of the new binary to create. Must match the name of a
+/// valid helper, as the binary name is used to select it.
+static void
+copy_tap_helper(const atf::tests::tc* tc, const char* name)
+{
+ const fs::path srcdir(tc->get_config_var("srcdir"));
+ atf::utils::copy_file((srcdir / "tap_helpers").str(), name);
+}
+
+
+/// Runs one tap test program and checks its result.
+///
+/// \param tc Pointer to the calling test case, to obtain srcdir.
+/// \param test_case_name Name of the "test case" to select from the helper
+/// program.
+/// \param exp_result The expected result.
+/// \param metadata The test case metadata.
+/// \param user_config User-provided configuration variables.
+static void
+run_one(const atf::tests::tc* tc, const char* test_case_name,
+ const model::test_result& exp_result,
+ const model::metadata& metadata = model::metadata_builder().build(),
+ const config::tree& user_config = engine::empty_config())
+{
+ copy_tap_helper(tc, test_case_name);
+ const model::test_program_ptr program = model::test_program_builder(
+ "tap", fs::path(test_case_name), fs::current_path(), "the-suite")
+ .add_test_case("main", metadata).build_ptr();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+ (void)handle.spawn_test(program, "main", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: ");
+ atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: ");
+ ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result());
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+} // anonymous namespace
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(list);
+ATF_TEST_CASE_BODY(list)
+{
+ const model::test_program program = model::test_program_builder(
+ "tap", fs::path("non-existent"), fs::path("."), "unused-suite")
+ .build();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+ const model::test_cases_map test_cases = handle.list_tests(
+ &program, engine::empty_config());
+ handle.cleanup();
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("main").build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__all_tests_pass);
+ATF_TEST_CASE_BODY(test__all_tests_pass)
+{
+ const model::test_result exp_result(model::test_result_passed);
+ run_one(this, "pass", exp_result);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__some_tests_fail);
+ATF_TEST_CASE_BODY(test__some_tests_fail)
+{
+ const model::test_result exp_result(model::test_result_failed,
+ "2 of 5 tests failed");
+ run_one(this, "fail", exp_result);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__all_tests_pass_but_exit_failure);
+ATF_TEST_CASE_BODY(test__all_tests_pass_but_exit_failure)
+{
+ const model::test_result exp_result(
+ model::test_result_broken,
+ "Dubious test program: reported all tests as passed but returned exit "
+ "code 70");
+ run_one(this, "pass_but_exit_failure", exp_result);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__signal_is_broken);
+ATF_TEST_CASE_BODY(test__signal_is_broken)
+{
+ const model::test_result exp_result(model::test_result_broken,
+ F("Received signal %s") % SIGABRT);
+ run_one(this, "crash", exp_result);
+}
+
+
+ATF_TEST_CASE(test__timeout_is_broken);
+ATF_TEST_CASE_HEAD(test__timeout_is_broken)
+{
+ set_md_var("timeout", "60");
+}
+ATF_TEST_CASE_BODY(test__timeout_is_broken)
+{
+ utils::setenv("CONTROL_DIR", fs::current_path().str());
+
+ const model::metadata metadata = model::metadata_builder()
+ .set_timeout(datetime::delta(1, 0)).build();
+ const model::test_result exp_result(model::test_result_broken,
+ "Test case timed out");
+ run_one(this, "timeout", exp_result, metadata);
+
+ ATF_REQUIRE(!atf::utils::file_exists("cookie"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(test__configuration_variables);
+ATF_TEST_CASE_BODY(test__configuration_variables)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.a-suite.first", "unused");
+ user_config.set_string("test_suites.the-suite.first", "some value");
+ user_config.set_string("test_suites.the-suite.second", "some other value");
+ user_config.set_string("test_suites.other-suite.first", "unused");
+
+ const model::test_result exp_result(model::test_result_passed);
+ run_one(this, "check_configuration_variables", exp_result,
+ model::metadata_builder().build(), user_config);
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ scheduler::register_interface(
+ "tap", std::shared_ptr< scheduler::interface >(
+ new engine::tap_interface()));
+
+ ATF_ADD_TEST_CASE(tcs, list);
+
+ ATF_ADD_TEST_CASE(tcs, test__all_tests_pass);
+ ATF_ADD_TEST_CASE(tcs, test__all_tests_pass_but_exit_failure);
+ ATF_ADD_TEST_CASE(tcs, test__some_tests_fail);
+ ATF_ADD_TEST_CASE(tcs, test__signal_is_broken);
+ ATF_ADD_TEST_CASE(tcs, test__timeout_is_broken);
+ ATF_ADD_TEST_CASE(tcs, test__configuration_variables);
+}