diff options
author | Brooks Davis <brooks@FreeBSD.org> | 2020-03-17 16:56:50 +0000 |
---|---|---|
committer | Brooks Davis <brooks@FreeBSD.org> | 2020-03-17 16:56:50 +0000 |
commit | 08334c51dbb99d9ecd2bb86a2d94ed06da9e167a (patch) | |
tree | c43eb24d59bd5c963583a5190caef80fc8387322 /engine | |
download | src-08334c51dbb99d9ecd2bb86a2d94ed06da9e167a.tar.gz src-08334c51dbb99d9ecd2bb86a2d94ed06da9e167a.zip |
Import the kyua testing framework for infrastructure softwarevendor/kyua/0.13-a685f91vendor/kyua
Imported at 0.13 plus assumulated changes to git hash a685f91.
Obtained from: https://github.com/jmmv/kyua
Sponsored by: DARPA
Notes
Notes:
svn path=/vendor/kyua/dist/; revision=359042
svn path=/vendor/kyua/0.13-a685f91/; revision=359043; tag=vendor/kyua/0.13-a685f91
Diffstat (limited to 'engine')
51 files changed, 13878 insertions, 0 deletions
diff --git a/engine/Kyuafile b/engine/Kyuafile new file mode 100644 index 000000000000..1baa63bc9118 --- /dev/null +++ b/engine/Kyuafile @@ -0,0 +1,17 @@ +syntax(2) + +test_suite("kyua") + +atf_test_program{name="atf_test"} +atf_test_program{name="atf_list_test"} +atf_test_program{name="atf_result_test"} +atf_test_program{name="config_test"} +atf_test_program{name="exceptions_test"} +atf_test_program{name="filters_test"} +atf_test_program{name="kyuafile_test"} +atf_test_program{name="plain_test"} +atf_test_program{name="requirements_test"} +atf_test_program{name="scanner_test"} +atf_test_program{name="tap_test"} +atf_test_program{name="tap_parser_test"} +atf_test_program{name="scheduler_test"} diff --git a/engine/Makefile.am.inc b/engine/Makefile.am.inc new file mode 100644 index 000000000000..baa7fe0bb8a0 --- /dev/null +++ b/engine/Makefile.am.inc @@ -0,0 +1,155 @@ +# Copyright 2010 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +ENGINE_CFLAGS = $(STORE_CFLAGS) $(MODEL_CFLAGS) $(UTILS_CFLAGS) +ENGINE_LIBS = libengine.a $(STORE_LIBS) $(MODEL_LIBS) $(UTILS_LIBS) + +noinst_LIBRARIES += libengine.a +libengine_a_CPPFLAGS = $(STORE_CFLAGS) $(UTILS_CFLAGS) +libengine_a_SOURCES = engine/atf.cpp +libengine_a_SOURCES += engine/atf.hpp +libengine_a_SOURCES += engine/atf_list.cpp +libengine_a_SOURCES += engine/atf_list.hpp +libengine_a_SOURCES += engine/atf_result.cpp +libengine_a_SOURCES += engine/atf_result.hpp +libengine_a_SOURCES += engine/atf_result_fwd.hpp +libengine_a_SOURCES += engine/config.cpp +libengine_a_SOURCES += engine/config.hpp +libengine_a_SOURCES += engine/config_fwd.hpp +libengine_a_SOURCES += engine/exceptions.cpp +libengine_a_SOURCES += engine/exceptions.hpp +libengine_a_SOURCES += engine/filters.cpp +libengine_a_SOURCES += engine/filters.hpp +libengine_a_SOURCES += engine/filters_fwd.hpp +libengine_a_SOURCES += engine/kyuafile.cpp +libengine_a_SOURCES += engine/kyuafile.hpp +libengine_a_SOURCES += engine/kyuafile_fwd.hpp +libengine_a_SOURCES += engine/plain.cpp +libengine_a_SOURCES += engine/plain.hpp +libengine_a_SOURCES += engine/requirements.cpp +libengine_a_SOURCES += engine/requirements.hpp +libengine_a_SOURCES += engine/scanner.cpp +libengine_a_SOURCES += engine/scanner.hpp +libengine_a_SOURCES += engine/scanner_fwd.hpp +libengine_a_SOURCES += engine/tap.cpp +libengine_a_SOURCES += engine/tap.hpp +libengine_a_SOURCES += engine/tap_parser.cpp +libengine_a_SOURCES += engine/tap_parser.hpp +libengine_a_SOURCES += engine/tap_parser_fwd.hpp +libengine_a_SOURCES += engine/scheduler.cpp +libengine_a_SOURCES += engine/scheduler.hpp +libengine_a_SOURCES += engine/scheduler_fwd.hpp + +if WITH_ATF +tests_enginedir = $(pkgtestsdir)/engine + +tests_engine_DATA = engine/Kyuafile +EXTRA_DIST += $(tests_engine_DATA) + +tests_engine_PROGRAMS = engine/atf_helpers +engine_atf_helpers_SOURCES = engine/atf_helpers.cpp +engine_atf_helpers_CXXFLAGS = $(UTILS_CFLAGS) $(ATF_CXX_CFLAGS) +engine_atf_helpers_LDADD = $(UTILS_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/atf_test +engine_atf_test_SOURCES = engine/atf_test.cpp +engine_atf_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_atf_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/atf_list_test +engine_atf_list_test_SOURCES = engine/atf_list_test.cpp +engine_atf_list_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_atf_list_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/atf_result_test +engine_atf_result_test_SOURCES = engine/atf_result_test.cpp +engine_atf_result_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_atf_result_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/config_test +engine_config_test_SOURCES = engine/config_test.cpp +engine_config_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_config_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/exceptions_test +engine_exceptions_test_SOURCES = engine/exceptions_test.cpp +engine_exceptions_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_exceptions_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/filters_test +engine_filters_test_SOURCES = engine/filters_test.cpp +engine_filters_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_filters_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/kyuafile_test +engine_kyuafile_test_SOURCES = engine/kyuafile_test.cpp +engine_kyuafile_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_kyuafile_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/plain_helpers +engine_plain_helpers_SOURCES = engine/plain_helpers.cpp +engine_plain_helpers_CXXFLAGS = $(UTILS_CFLAGS) +engine_plain_helpers_LDADD = $(UTILS_LIBS) + +tests_engine_PROGRAMS += engine/plain_test +engine_plain_test_SOURCES = engine/plain_test.cpp +engine_plain_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_plain_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/requirements_test +engine_requirements_test_SOURCES = engine/requirements_test.cpp +engine_requirements_test_CXXFLAGS = $(ENGINE_CFLAGS) $(UTILS_TEST_CFLAGS) \ + $(ATF_CXX_CFLAGS) +engine_requirements_test_LDADD = $(ENGINE_LIBS) $(UTILS_TEST_LIBS) \ + $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/scanner_test +engine_scanner_test_SOURCES = engine/scanner_test.cpp +engine_scanner_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_scanner_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/tap_helpers +engine_tap_helpers_SOURCES = engine/tap_helpers.cpp +engine_tap_helpers_CXXFLAGS = $(UTILS_CFLAGS) +engine_tap_helpers_LDADD = $(UTILS_LIBS) + +tests_engine_PROGRAMS += engine/tap_test +engine_tap_test_SOURCES = engine/tap_test.cpp +engine_tap_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_tap_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/tap_parser_test +engine_tap_parser_test_SOURCES = engine/tap_parser_test.cpp +engine_tap_parser_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_tap_parser_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) + +tests_engine_PROGRAMS += engine/scheduler_test +engine_scheduler_test_SOURCES = engine/scheduler_test.cpp +engine_scheduler_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) +engine_scheduler_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) +endif diff --git a/engine/atf.cpp b/engine/atf.cpp new file mode 100644 index 000000000000..eb63be20b0e7 --- /dev/null +++ b/engine/atf.cpp @@ -0,0 +1,242 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/atf.hpp" + +extern "C" { +#include <unistd.h> +} + +#include <cerrno> +#include <cstdlib> +#include <fstream> + +#include "engine/atf_list.hpp" +#include "engine/atf_result.hpp" +#include "engine/exceptions.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/defs.hpp" +#include "utils/env.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/logging/macros.hpp" +#include "utils/optional.ipp" +#include "utils/process/exceptions.hpp" +#include "utils/process/operations.hpp" +#include "utils/process/status.hpp" +#include "utils/stream.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace process = utils::process; + +using utils::optional; + + +namespace { + + +/// Basename of the file containing the result written by the ATF test case. +static const char* result_name = "result.atf"; + + +/// Magic numbers returned by exec_list when exec(2) fails. +enum list_exit_code { + exit_eacces = 90, + exit_enoent, + exit_enoexec, +}; + + +} // anonymous namespace + + +/// Executes a test program's list operation. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +/// +/// \param test_program The test program to execute. +/// \param vars User-provided variables to pass to the test program. +void +engine::atf_interface::exec_list(const model::test_program& test_program, + const config::properties_map& vars) const +{ + utils::setenv("__RUNNING_INSIDE_ATF_RUN", "internal-yes-value"); + + process::args_vector args; + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + args.push_back(F("-v%s=%s") % (*iter).first % (*iter).second); + } + + args.push_back("-l"); + try { + process::exec_unsafe(test_program.absolute_path(), args); + } catch (const process::system_error& e) { + if (e.original_errno() == EACCES) + ::_exit(exit_eacces); + else if (e.original_errno() == ENOENT) + ::_exit(exit_enoent); + else if (e.original_errno() == ENOEXEC) + ::_exit(exit_enoexec); + throw; + } +} + + +/// Computes the test cases list of a test program. +/// +/// \param status The termination status of the subprocess used to execute +/// the exec_test() method or none if the test timed out. +/// \param stdout_path Path to the file containing the stdout of the test. +/// \param stderr_path Path to the file containing the stderr of the test. +/// +/// \return A list of test cases. +/// +/// \throw error If there is a problem parsing the test case list. +model::test_cases_map +engine::atf_interface::parse_list(const optional< process::status >& status, + const fs::path& stdout_path, + const fs::path& stderr_path) const +{ + const std::string stderr_contents = utils::read_file(stderr_path); + if (!stderr_contents.empty()) + LW("Test case list wrote to stderr: " + stderr_contents); + + if (!status) + throw engine::error("Test case list timed out"); + if (status.get().exited()) { + const int exitstatus = status.get().exitstatus(); + if (exitstatus == EXIT_SUCCESS) { + // Nothing to do; fall through. + } else if (exitstatus == exit_eacces) { + throw engine::error("Permission denied to run test program"); + } else if (exitstatus == exit_enoent) { + throw engine::error("Cannot find test program"); + } else if (exitstatus == exit_enoexec) { + throw engine::error("Invalid test program format"); + } else { + throw engine::error("Test program did not exit cleanly"); + } + } else { + throw engine::error("Test program received signal"); + } + + std::ifstream input(stdout_path.c_str()); + if (!input) + throw engine::load_error(stdout_path, "Cannot open file for read"); + const model::test_cases_map test_cases = parse_atf_list(input); + + if (!stderr_contents.empty()) + throw engine::error("Test case list wrote to stderr"); + + return test_cases; +} + + +/// Executes a test case of the test program. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +/// +/// \param test_program The test program to execute. +/// \param test_case_name Name of the test case to invoke. +/// \param vars User-provided variables to pass to the test program. +/// \param control_directory Directory where the interface may place control +/// files. +void +engine::atf_interface::exec_test(const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars, + const fs::path& control_directory) const +{ + utils::setenv("__RUNNING_INSIDE_ATF_RUN", "internal-yes-value"); + + process::args_vector args; + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + args.push_back(F("-v%s=%s") % (*iter).first % (*iter).second); + } + + args.push_back(F("-r%s") % (control_directory / result_name)); + args.push_back(test_case_name); + process::exec(test_program.absolute_path(), args); +} + + +/// Executes a test cleanup routine of the test program. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +/// +/// \param test_program The test program to execute. +/// \param test_case_name Name of the test case to invoke. +/// \param vars User-provided variables to pass to the test program. +void +engine::atf_interface::exec_cleanup( + const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars, + const fs::path& /* control_directory */) const +{ + utils::setenv("__RUNNING_INSIDE_ATF_RUN", "internal-yes-value"); + + process::args_vector args; + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + args.push_back(F("-v%s=%s") % (*iter).first % (*iter).second); + } + + args.push_back(F("%s:cleanup") % test_case_name); + process::exec(test_program.absolute_path(), args); +} + + +/// Computes the result of a test case based on its termination status. +/// +/// \param status The termination status of the subprocess used to execute +/// the exec_test() method or none if the test timed out. +/// \param control_directory Directory where the interface may have placed +/// control files. +/// +/// \return A test result. +model::test_result +engine::atf_interface::compute_result( + const optional< process::status >& status, + const fs::path& control_directory, + const fs::path& /* stdout_path */, + const fs::path& /* stderr_path */) const +{ + return calculate_atf_result(status, control_directory / result_name); +} diff --git a/engine/atf.hpp b/engine/atf.hpp new file mode 100644 index 000000000000..34ddc2413235 --- /dev/null +++ b/engine/atf.hpp @@ -0,0 +1,72 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/atf.hpp +/// Execution engine for test programs that implement the atf interface. + +#if !defined(ENGINE_ATF_HPP) +#define ENGINE_ATF_HPP + +#include "engine/scheduler.hpp" + +namespace engine { + + +/// Implementation of the scheduler interface for atf test programs. +class atf_interface : public engine::scheduler::interface { +public: + void exec_list(const model::test_program&, + const utils::config::properties_map&) const UTILS_NORETURN; + + model::test_cases_map parse_list( + const utils::optional< utils::process::status >&, + const utils::fs::path&, + const utils::fs::path&) const; + + void exec_test(const model::test_program&, const std::string&, + const utils::config::properties_map&, + const utils::fs::path&) const + UTILS_NORETURN; + + void exec_cleanup(const model::test_program&, const std::string&, + const utils::config::properties_map&, + const utils::fs::path&) const + UTILS_NORETURN; + + model::test_result compute_result( + const utils::optional< utils::process::status >&, + const utils::fs::path&, + const utils::fs::path&, + const utils::fs::path&) const; +}; + + +} // namespace engine + + +#endif // !defined(ENGINE_ATF_HPP) diff --git a/engine/atf_helpers.cpp b/engine/atf_helpers.cpp new file mode 100644 index 000000000000..c45654f10e58 --- /dev/null +++ b/engine/atf_helpers.cpp @@ -0,0 +1,414 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include <sys/stat.h> + +#include <signal.h> +#include <unistd.h> +} + +#include <algorithm> +#include <cstdlib> +#include <fstream> +#include <iostream> +#include <set> +#include <sstream> +#include <string> +#include <vector> + +#include <atf-c++.hpp> + +#include "utils/env.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/logging/operations.hpp" +#include "utils/optional.ipp" +#include "utils/test_utils.ipp" +#include "utils/text/operations.ipp" + +namespace fs = utils::fs; +namespace logging = utils::logging; +namespace text = utils::text; + +using utils::optional; + + +namespace { + + +/// Creates an empty file in the given directory. +/// +/// \param test_case The test case currently running. +/// \param directory The name of the configuration variable that holds the path +/// to the directory in which to create the cookie file. +/// \param name The name of the cookie file to create. +static void +create_cookie(const atf::tests::tc* test_case, const char* directory, + const char* name) +{ + if (!test_case->has_config_var(directory)) + test_case->fail(std::string(name) + " not provided"); + + const fs::path control_dir(test_case->get_config_var(directory)); + std::ofstream file((control_dir / name).c_str()); + if (!file) + test_case->fail("Failed to create the control cookie"); + file.close(); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITH_CLEANUP(check_cleanup_workdir); +ATF_TEST_CASE_HEAD(check_cleanup_workdir) +{ + set_md_var("require.config", "control_dir"); +} +ATF_TEST_CASE_BODY(check_cleanup_workdir) +{ + std::ofstream cookie("workdir_cookie"); + cookie << "1234\n"; + cookie.close(); + skip("cookie created"); +} +ATF_TEST_CASE_CLEANUP(check_cleanup_workdir) +{ + const fs::path control_dir(get_config_var("control_dir")); + + std::ifstream cookie("workdir_cookie"); + if (!cookie) { + std::ofstream((control_dir / "missing_cookie").c_str()).close(); + std::exit(EXIT_FAILURE); + } + + std::string value; + cookie >> value; + if (value != "1234") { + std::ofstream((control_dir / "invalid_cookie").c_str()).close(); + std::exit(EXIT_FAILURE); + } + + std::ofstream((control_dir / "cookie_ok").c_str()).close(); + std::exit(EXIT_SUCCESS); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_configuration_variables); +ATF_TEST_CASE_BODY(check_configuration_variables) +{ + ATF_REQUIRE(has_config_var("first")); + ATF_REQUIRE_EQ("some value", get_config_var("first")); + + ATF_REQUIRE(has_config_var("second")); + ATF_REQUIRE_EQ("some other value", get_config_var("second")); +} + + +ATF_TEST_CASE(check_list_config); +ATF_TEST_CASE_HEAD(check_list_config) +{ + std::string description = "Found:"; + + if (has_config_var("var1")) + description += " var1=" + get_config_var("var1"); + if (has_config_var("var2")) + description += " var2=" + get_config_var("var2"); + + set_md_var("descr", description); +} +ATF_TEST_CASE_BODY(check_list_config) +{ +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_unprivileged); +ATF_TEST_CASE_BODY(check_unprivileged) +{ + if (::getuid() == 0) + fail("Running as root, but I shouldn't be"); + + std::ofstream file("cookie"); + if (!file) + fail("Failed to create the cookie; work directory probably owned by " + "root"); + file.close(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(crash); +ATF_TEST_CASE_BODY(crash) +{ + std::abort(); +} + + +ATF_TEST_CASE(crash_head); +ATF_TEST_CASE_HEAD(crash_head) +{ + utils::abort_without_coredump(); +} +ATF_TEST_CASE_BODY(crash_head) +{ +} + + +ATF_TEST_CASE_WITH_CLEANUP(crash_cleanup); +ATF_TEST_CASE_HEAD(crash_cleanup) +{ +} +ATF_TEST_CASE_BODY(crash_cleanup) +{ +} +ATF_TEST_CASE_CLEANUP(crash_cleanup) +{ + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(create_cookie_in_control_dir); +ATF_TEST_CASE_BODY(create_cookie_in_control_dir) +{ + create_cookie(this, "control_dir", "cookie"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(create_cookie_in_workdir); +ATF_TEST_CASE_BODY(create_cookie_in_workdir) +{ + std::ofstream file("cookie"); + if (!file) + fail("Failed to create the cookie"); + file.close(); +} + + +ATF_TEST_CASE_WITH_CLEANUP(create_cookie_from_cleanup); +ATF_TEST_CASE_HEAD(create_cookie_from_cleanup) +{ +} +ATF_TEST_CASE_BODY(create_cookie_from_cleanup) +{ +} +ATF_TEST_CASE_CLEANUP(create_cookie_from_cleanup) +{ + create_cookie(this, "control_dir", "cookie"); +} + + +ATF_TEST_CASE_WITH_CLEANUP(expect_timeout); +ATF_TEST_CASE_HEAD(expect_timeout) +{ + if (has_config_var("timeout")) + set_md_var("timeout", get_config_var("timeout")); +} +ATF_TEST_CASE_BODY(expect_timeout) +{ + expect_timeout("Times out on purpose"); + ::sleep(10); + create_cookie(this, "control_dir", "cookie"); +} +ATF_TEST_CASE_CLEANUP(expect_timeout) +{ + create_cookie(this, "control_dir", "cookie.cleanup"); +} + + +ATF_TEST_CASE_WITH_CLEANUP(output); +ATF_TEST_CASE_HEAD(output) +{ +} +ATF_TEST_CASE_BODY(output) +{ + std::cout << "Body message to stdout\n"; + std::cerr << "Body message to stderr\n"; +} +ATF_TEST_CASE_CLEANUP(output) +{ + std::cout << "Cleanup message to stdout\n"; + std::cerr << "Cleanup message to stderr\n"; +} + + +ATF_TEST_CASE(output_in_list); +ATF_TEST_CASE_HEAD(output_in_list) +{ + std::cerr << "Should not write anything!\n"; +} +ATF_TEST_CASE_BODY(output_in_list) +{ +} + + +ATF_TEST_CASE(pass); +ATF_TEST_CASE_HEAD(pass) +{ + set_md_var("descr", "Always-passing test case"); +} +ATF_TEST_CASE_BODY(pass) +{ +} + + +ATF_TEST_CASE_WITH_CLEANUP(shared_workdir); +ATF_TEST_CASE_HEAD(shared_workdir) +{ +} +ATF_TEST_CASE_BODY(shared_workdir) +{ + atf::utils::create_file("shared_cookie", ""); +} +ATF_TEST_CASE_CLEANUP(shared_workdir) +{ + if (!atf::utils::file_exists("shared_cookie")) + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE(spawn_blocking_child); +ATF_TEST_CASE_HEAD(spawn_blocking_child) +{ + set_md_var("require.config", "control_dir"); +} +ATF_TEST_CASE_BODY(spawn_blocking_child) +{ + pid_t pid = ::fork(); + if (pid == -1) + fail("Cannot fork subprocess"); + else if (pid == 0) { + for (;;) + ::pause(); + } else { + const fs::path name = fs::path(get_config_var("control_dir")) / "pid"; + std::ofstream pidfile(name.c_str()); + ATF_REQUIRE(pidfile); + pidfile << pid; + pidfile.close(); + } +} + + +ATF_TEST_CASE_WITH_CLEANUP(timeout_body); +ATF_TEST_CASE_HEAD(timeout_body) +{ + if (has_config_var("timeout")) + set_md_var("timeout", get_config_var("timeout")); +} +ATF_TEST_CASE_BODY(timeout_body) +{ + ::sleep(10); + create_cookie(this, "control_dir", "cookie"); +} +ATF_TEST_CASE_CLEANUP(timeout_body) +{ + create_cookie(this, "control_dir", "cookie.cleanup"); +} + + +ATF_TEST_CASE_WITH_CLEANUP(timeout_cleanup); +ATF_TEST_CASE_HEAD(timeout_cleanup) +{ +} +ATF_TEST_CASE_BODY(timeout_cleanup) +{ +} +ATF_TEST_CASE_CLEANUP(timeout_cleanup) +{ + ::sleep(10); + create_cookie(this, "control_dir", "cookie"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(validate_isolation); +ATF_TEST_CASE_BODY(validate_isolation) +{ + ATF_REQUIRE(utils::getenv("HOME").get() != "fake-value"); + ATF_REQUIRE(!utils::getenv("LANG")); +} + + +/// Wrapper around ATF_ADD_TEST_CASE to only add a test when requested. +/// +/// The caller can set the TEST_CASES environment variable to a +/// whitespace-separated list of test case names to enable. If not empty, the +/// list acts as a filter for the tests to add. +/// +/// \param tcs List of test cases into which to register the test. +/// \param filters List of filters to determine whether the test applies or not. +/// \param name Name of the test case being added. +#define ADD_TEST_CASE(tcs, filters, name) \ + do { \ + if (filters.empty() || filters.find(#name) != filters.end()) \ + ATF_ADD_TEST_CASE(tcs, name); \ + } while (false) + + +ATF_INIT_TEST_CASES(tcs) +{ + logging::set_inmemory(); + + // TODO(jmmv): Instead of using "filters", we should make TEST_CASES + // explicitly list all the test cases to enable. This would let us get rid + // of some of the hacks below... + std::set< std::string > filters; + + const optional< std::string > names_raw = utils::getenv("TEST_CASES"); + if (names_raw) { + if (names_raw.get().empty()) + return; // See TODO above. + + const std::vector< std::string > names = text::split( + names_raw.get(), ' '); + std::copy(names.begin(), names.end(), + std::inserter(filters, filters.begin())); + } + + if (filters.find("crash_head") != filters.end()) // See TODO above. + ATF_ADD_TEST_CASE(tcs, crash_head); + if (filters.find("output_in_list") != filters.end()) // See TODO above. + ATF_ADD_TEST_CASE(tcs, output_in_list); + + ADD_TEST_CASE(tcs, filters, check_cleanup_workdir); + ADD_TEST_CASE(tcs, filters, check_configuration_variables); + ADD_TEST_CASE(tcs, filters, check_list_config); + ADD_TEST_CASE(tcs, filters, check_unprivileged); + ADD_TEST_CASE(tcs, filters, crash); + ADD_TEST_CASE(tcs, filters, crash_cleanup); + ADD_TEST_CASE(tcs, filters, create_cookie_in_control_dir); + ADD_TEST_CASE(tcs, filters, create_cookie_in_workdir); + ADD_TEST_CASE(tcs, filters, create_cookie_from_cleanup); + ADD_TEST_CASE(tcs, filters, expect_timeout); + ADD_TEST_CASE(tcs, filters, output); + ADD_TEST_CASE(tcs, filters, pass); + ADD_TEST_CASE(tcs, filters, shared_workdir); + ADD_TEST_CASE(tcs, filters, spawn_blocking_child); + ADD_TEST_CASE(tcs, filters, timeout_body); + ADD_TEST_CASE(tcs, filters, timeout_cleanup); + ADD_TEST_CASE(tcs, filters, validate_isolation); +} diff --git a/engine/atf_list.cpp b/engine/atf_list.cpp new file mode 100644 index 000000000000..a16b889c74f0 --- /dev/null +++ b/engine/atf_list.cpp @@ -0,0 +1,196 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/atf_list.hpp" + +#include <fstream> +#include <string> +#include <utility> + +#include "engine/exceptions.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "utils/config/exceptions.hpp" +#include "utils/format/macros.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; + + +namespace { + + +/// Splits a property line of the form "name: word1 [... wordN]". +/// +/// \param line The line to parse. +/// +/// \return A (property_name, property_value) pair. +/// +/// \throw format_error If the value of line is invalid. +static std::pair< std::string, std::string > +split_prop_line(const std::string& line) +{ + const std::string::size_type pos = line.find(": "); + if (pos == std::string::npos) + throw engine::format_error("Invalid property line; expecting line of " + "the form 'name: value'"); + return std::make_pair(line.substr(0, pos), line.substr(pos + 2)); +} + + +/// Parses a set of consecutive property lines. +/// +/// Processing stops when an empty line or the end of file is reached. None of +/// these conditions indicate errors. +/// +/// \param input The stream to read the lines from. +/// +/// \return The parsed property lines. +/// +/// throw format_error If the input stream has an invalid format. +static model::properties_map +parse_properties(std::istream& input) +{ + model::properties_map properties; + + std::string line; + while (std::getline(input, line).good() && !line.empty()) { + const std::pair< std::string, std::string > property = split_prop_line( + line); + if (properties.find(property.first) != properties.end()) + throw engine::format_error("Duplicate value for property " + + property.first); + properties.insert(property); + } + + return properties; +} + + +} // anonymous namespace + + +/// Parses the metadata of an ATF test case. +/// +/// \param props The properties (name/value string pairs) as provided by the +/// ATF test program. +/// +/// \return A parsed metadata object. +/// +/// \throw engine::format_error If the syntax of any of the properties is +/// invalid. +model::metadata +engine::parse_atf_metadata(const model::properties_map& props) +{ + model::metadata_builder mdbuilder; + + try { + for (model::properties_map::const_iterator iter = props.begin(); + iter != props.end(); iter++) { + const std::string& name = (*iter).first; + const std::string& value = (*iter).second; + + if (name == "descr") { + mdbuilder.set_string("description", value); + } else if (name == "has.cleanup") { + mdbuilder.set_string("has_cleanup", value); + } else if (name == "require.arch") { + mdbuilder.set_string("allowed_architectures", value); + } else if (name == "require.config") { + mdbuilder.set_string("required_configs", value); + } else if (name == "require.files") { + mdbuilder.set_string("required_files", value); + } else if (name == "require.machine") { + mdbuilder.set_string("allowed_platforms", value); + } else if (name == "require.memory") { + mdbuilder.set_string("required_memory", value); + } else if (name == "require.progs") { + mdbuilder.set_string("required_programs", value); + } else if (name == "require.user") { + mdbuilder.set_string("required_user", value); + } else if (name == "timeout") { + mdbuilder.set_string("timeout", value); + } else if (name.length() > 2 && name.substr(0, 2) == "X-") { + mdbuilder.add_custom(name.substr(2), value); + } else { + throw engine::format_error(F("Unknown test case metadata " + "property '%s'") % name); + } + } + } catch (const config::error& e) { + throw engine::format_error(e.what()); + } + + return mdbuilder.build(); +} + + +/// Parses the ATF list of test cases from an open stream. +/// +/// \param input The stream to read from. +/// +/// \return The collection of parsed test cases. +/// +/// \throw format_error If there is any problem in the input data. +model::test_cases_map +engine::parse_atf_list(std::istream& input) +{ + std::string line; + + std::getline(input, line); + if (line != "Content-Type: application/X-atf-tp; version=\"1\"" + || !input.good()) + throw format_error(F("Invalid header for test case list; expecting " + "Content-Type for application/X-atf-tp version 1, " + "got '%s'") % line); + + std::getline(input, line); + if (!line.empty() || !input.good()) + throw format_error(F("Invalid header for test case list; expecting " + "a blank line, got '%s'") % line); + + model::test_cases_map_builder test_cases_builder; + while (std::getline(input, line).good()) { + const std::pair< std::string, std::string > ident = split_prop_line( + line); + if (ident.first != "ident" or ident.second.empty()) + throw format_error("Invalid test case definition; must be " + "preceeded by the identifier"); + + const model::properties_map props = parse_properties(input); + test_cases_builder.add(ident.second, parse_atf_metadata(props)); + } + const model::test_cases_map test_cases = test_cases_builder.build(); + if (test_cases.empty()) { + // The scheduler interface also checks for the presence of at least one + // test case. However, because the atf format itself requires one test + // case to be always present, we check for this condition here as well. + throw format_error("No test cases"); + } + return test_cases; +} diff --git a/engine/atf_list.hpp b/engine/atf_list.hpp new file mode 100644 index 000000000000..3d81d03e3bcf --- /dev/null +++ b/engine/atf_list.hpp @@ -0,0 +1,51 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/atf_list.hpp +/// Parser of ATF test case lists. + +#if !defined(ENGINE_ATF_LIST_HPP) +#define ENGINE_ATF_LIST_HPP + +#include <istream> + +#include "model/metadata_fwd.hpp" +#include "model/test_case_fwd.hpp" +#include "model/types.hpp" +#include "utils/fs/path_fwd.hpp" + +namespace engine { + + +model::metadata parse_atf_metadata(const model::properties_map&); +model::test_cases_map parse_atf_list(std::istream&); + + +} // namespace engine + +#endif // !defined(ENGINE_ATF_LIST_HPP) diff --git a/engine/atf_list_test.cpp b/engine/atf_list_test.cpp new file mode 100644 index 000000000000..7f19ca8fbec5 --- /dev/null +++ b/engine/atf_list_test.cpp @@ -0,0 +1,278 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/atf_list.hpp" + +#include <sstream> +#include <string> + +#include <atf-c++.hpp> + +#include "engine/exceptions.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/types.hpp" +#include "utils/datetime.hpp" +#include "utils/format/containers.ipp" +#include "utils/fs/path.hpp" +#include "utils/units.hpp" + +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace units = utils::units; + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_metadata__defaults) +ATF_TEST_CASE_BODY(parse_atf_metadata__defaults) +{ + const model::properties_map properties; + const model::metadata md = engine::parse_atf_metadata(properties); + + const model::metadata exp_md = model::metadata_builder().build(); + ATF_REQUIRE_EQ(exp_md, md); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_metadata__override_all) +ATF_TEST_CASE_BODY(parse_atf_metadata__override_all) +{ + model::properties_map properties; + properties["descr"] = "Some text"; + properties["has.cleanup"] = "true"; + properties["require.arch"] = "i386 x86_64"; + properties["require.config"] = "var1 var2 var3"; + properties["require.files"] = "/file1 /dir/file2"; + properties["require.machine"] = "amd64"; + properties["require.memory"] = "1m"; + properties["require.progs"] = "/bin/ls svn"; + properties["require.user"] = "root"; + properties["timeout"] = "123"; + properties["X-foo"] = "value1"; + properties["X-bar"] = "value2"; + properties["X-baz-www"] = "value3"; + const model::metadata md = engine::parse_atf_metadata(properties); + + const model::metadata exp_md = model::metadata_builder() + .add_allowed_architecture("i386") + .add_allowed_architecture("x86_64") + .add_allowed_platform("amd64") + .add_custom("foo", "value1") + .add_custom("bar", "value2") + .add_custom("baz-www", "value3") + .add_required_config("var1") + .add_required_config("var2") + .add_required_config("var3") + .add_required_file(fs::path("/file1")) + .add_required_file(fs::path("/dir/file2")) + .add_required_program(fs::path("/bin/ls")) + .add_required_program(fs::path("svn")) + .set_description("Some text") + .set_has_cleanup(true) + .set_required_memory(units::bytes::parse("1m")) + .set_required_user("root") + .set_timeout(datetime::delta(123, 0)) + .build(); + ATF_REQUIRE_EQ(exp_md, md); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_metadata__unknown) +ATF_TEST_CASE_BODY(parse_atf_metadata__unknown) +{ + model::properties_map properties; + properties["foobar"] = "Some text"; + + ATF_REQUIRE_THROW_RE(engine::format_error, "Unknown.*property.*'foobar'", + engine::parse_atf_metadata(properties)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__empty); +ATF_TEST_CASE_BODY(parse_atf_list__empty) +{ + const std::string text = ""; + std::istringstream input(text); + ATF_REQUIRE_THROW_RE(engine::format_error, "expecting Content-Type", + engine::parse_atf_list(input)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__invalid_header); +ATF_TEST_CASE_BODY(parse_atf_list__invalid_header) +{ + { + const std::string text = + "Content-Type: application/X-atf-tp; version=\"1\"\n"; + std::istringstream input(text); + ATF_REQUIRE_THROW_RE(engine::format_error, "expecting.*blank line", + engine::parse_atf_list(input)); + } + + { + const std::string text = + "Content-Type: application/X-atf-tp; version=\"1\"\nfoo\n"; + std::istringstream input(text); + ATF_REQUIRE_THROW_RE(engine::format_error, "expecting.*blank line", + engine::parse_atf_list(input)); + } + + { + const std::string text = + "Content-Type: application/X-atf-tp; version=\"2\"\n\n"; + std::istringstream input(text); + ATF_REQUIRE_THROW_RE(engine::format_error, "expecting Content-Type", + engine::parse_atf_list(input)); + } +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__no_test_cases); +ATF_TEST_CASE_BODY(parse_atf_list__no_test_cases) +{ + const std::string text = + "Content-Type: application/X-atf-tp; version=\"1\"\n\n"; + std::istringstream input(text); + ATF_REQUIRE_THROW_RE(engine::format_error, "No test cases", + engine::parse_atf_list(input)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__one_test_case_simple); +ATF_TEST_CASE_BODY(parse_atf_list__one_test_case_simple) +{ + const std::string text = + "Content-Type: application/X-atf-tp; version=\"1\"\n" + "\n" + "ident: test-case\n"; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_atf_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("test-case").build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__one_test_case_complex); +ATF_TEST_CASE_BODY(parse_atf_list__one_test_case_complex) +{ + const std::string text = + "Content-Type: application/X-atf-tp; version=\"1\"\n" + "\n" + "ident: first\n" + "descr: This is the description\n" + "timeout: 500\n"; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_atf_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("first", model::metadata_builder() + .set_description("This is the description") + .set_timeout(datetime::delta(500, 0)) + .build()) + .build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__one_test_case_invalid_syntax); +ATF_TEST_CASE_BODY(parse_atf_list__one_test_case_invalid_syntax) +{ + const std::string text = + "Content-Type: application/X-atf-tp; version=\"1\"\n\n" + "descr: This is the description\n" + "ident: first\n"; + std::istringstream input(text); + ATF_REQUIRE_THROW_RE(engine::format_error, "preceeded.*identifier", + engine::parse_atf_list(input)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__one_test_case_invalid_properties); +ATF_TEST_CASE_BODY(parse_atf_list__one_test_case_invalid_properties) +{ + // Inject a single invalid property that makes test_case::from_properties() + // raise a particular error message so that we can validate that such + // function was called. We do intensive testing separately, so it is not + // necessary to redo it here. + const std::string text = + "Content-Type: application/X-atf-tp; version=\"1\"\n\n" + "ident: first\n" + "require.progs: bin/ls\n"; + std::istringstream input(text); + ATF_REQUIRE_THROW_RE(engine::format_error, "Relative path 'bin/ls'", + engine::parse_atf_list(input)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_atf_list__many_test_cases); +ATF_TEST_CASE_BODY(parse_atf_list__many_test_cases) +{ + const std::string text = + "Content-Type: application/X-atf-tp; version=\"1\"\n" + "\n" + "ident: first\n" + "descr: This is the description\n" + "\n" + "ident: second\n" + "timeout: 500\n" + "descr: Some text\n" + "\n" + "ident: third\n"; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_atf_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("first", model::metadata_builder() + .set_description("This is the description") + .build()) + .add("second", model::metadata_builder() + .set_description("Some text") + .set_timeout(datetime::delta(500, 0)) + .build()) + .add("third") + .build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, parse_atf_metadata__defaults); + ATF_ADD_TEST_CASE(tcs, parse_atf_metadata__override_all); + ATF_ADD_TEST_CASE(tcs, parse_atf_metadata__unknown); + + ATF_ADD_TEST_CASE(tcs, parse_atf_list__empty); + ATF_ADD_TEST_CASE(tcs, parse_atf_list__invalid_header); + ATF_ADD_TEST_CASE(tcs, parse_atf_list__no_test_cases); + ATF_ADD_TEST_CASE(tcs, parse_atf_list__one_test_case_simple); + ATF_ADD_TEST_CASE(tcs, parse_atf_list__one_test_case_complex); + ATF_ADD_TEST_CASE(tcs, parse_atf_list__one_test_case_invalid_syntax); + ATF_ADD_TEST_CASE(tcs, parse_atf_list__one_test_case_invalid_properties); + ATF_ADD_TEST_CASE(tcs, parse_atf_list__many_test_cases); +} diff --git a/engine/atf_result.cpp b/engine/atf_result.cpp new file mode 100644 index 000000000000..f99b28f9e96e --- /dev/null +++ b/engine/atf_result.cpp @@ -0,0 +1,642 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/atf_result.hpp" + +#include <cstdlib> +#include <fstream> +#include <utility> + +#include "engine/exceptions.hpp" +#include "model/test_result.hpp" +#include "utils/fs/path.hpp" +#include "utils/format/macros.hpp" +#include "utils/optional.ipp" +#include "utils/process/status.hpp" +#include "utils/sanity.hpp" +#include "utils/text/exceptions.hpp" +#include "utils/text/operations.ipp" + +namespace fs = utils::fs; +namespace process = utils::process; +namespace text = utils::text; + +using utils::none; +using utils::optional; + + +namespace { + + +/// Reads a file and flattens its lines. +/// +/// The main purpose of this function is to simplify the parsing of a file +/// containing the result of a test. Therefore, the return value carries +/// several assumptions. +/// +/// \param input The stream to read from. +/// +/// \return A pair (line count, contents) detailing how many lines where read +/// and their contents. If the file contains a single line with no newline +/// character, the line count is 0. If the file includes more than one line, +/// the lines are merged together and separated by the magic string +/// '<<NEWLINE>>'. +static std::pair< size_t, std::string > +read_lines(std::istream& input) +{ + std::pair< size_t, std::string > ret = std::make_pair(0, ""); + + do { + std::string line; + std::getline(input, line); + if (input.eof() && !line.empty()) { + if (ret.first == 0) + ret.second = line; + else { + ret.second += "<<NEWLINE>>" + line; + ret.first++; + } + } else if (input.good()) { + if (ret.first == 0) + ret.second = line; + else + ret.second += "<<NEWLINE>>" + line; + ret.first++; + } + } while (input.good()); + + return ret; +} + + +/// Parses a test result that does not accept a reason. +/// +/// \param status The result status name. +/// \param rest The rest of the line after the status name. +/// +/// \return An object representing the test result. +/// +/// \throw format_error If the result is invalid (i.e. rest is invalid). +/// +/// \pre status must be "passed". +static engine::atf_result +parse_without_reason(const std::string& status, const std::string& rest) +{ + if (!rest.empty()) + throw engine::format_error(F("%s cannot have a reason") % status); + PRE(status == "passed"); + return engine::atf_result(engine::atf_result::passed); +} + + +/// Parses a test result that needs a reason. +/// +/// \param status The result status name. +/// \param rest The rest of the line after the status name. +/// +/// \return An object representing the test result. +/// +/// \throw format_error If the result is invalid (i.e. rest is invalid). +/// +/// \pre status must be one of "broken", "expected_death", "expected_failure", +/// "expected_timeout", "failed" or "skipped". +static engine::atf_result +parse_with_reason(const std::string& status, const std::string& rest) +{ + using engine::atf_result; + + if (rest.length() < 3 || rest.substr(0, 2) != ": ") + throw engine::format_error(F("%s must be followed by ': <reason>'") % + status); + const std::string reason = rest.substr(2); + INV(!reason.empty()); + + if (status == "broken") + return atf_result(atf_result::broken, reason); + else if (status == "expected_death") + return atf_result(atf_result::expected_death, reason); + else if (status == "expected_failure") + return atf_result(atf_result::expected_failure, reason); + else if (status == "expected_timeout") + return atf_result(atf_result::expected_timeout, reason); + else if (status == "failed") + return atf_result(atf_result::failed, reason); + else if (status == "skipped") + return atf_result(atf_result::skipped, reason); + else + PRE_MSG(false, "Unexpected status"); +} + + +/// Converts a string to an integer. +/// +/// \param str The string containing the integer to convert. +/// +/// \return The converted integer; none if the parsing fails. +static optional< int > +parse_int(const std::string& str) +{ + try { + return utils::make_optional(text::to_type< int >(str)); + } catch (const text::value_error& e) { + return none; + } +} + + +/// Parses a test result that needs a reason and accepts an optional integer. +/// +/// \param status The result status name. +/// \param rest The rest of the line after the status name. +/// +/// \return The parsed test result if the data is valid, or a broken result if +/// the parsing failed. +/// +/// \pre status must be one of "expected_exit" or "expected_signal". +static engine::atf_result +parse_with_reason_and_arg(const std::string& status, const std::string& rest) +{ + using engine::atf_result; + + std::string::size_type delim = rest.find_first_of(":("); + if (delim == std::string::npos) + throw engine::format_error(F("Invalid format for '%s' test case " + "result; must be followed by '[(num)]: " + "<reason>' but found '%s'") % + status % rest); + + optional< int > arg; + if (rest[delim] == '(') { + const std::string::size_type delim2 = rest.find("):", delim); + if (delim == std::string::npos) + throw engine::format_error(F("Mismatched '(' in %s") % rest); + + const std::string argstr = rest.substr(delim + 1, delim2 - delim - 1); + arg = parse_int(argstr); + if (!arg) + throw engine::format_error(F("Invalid integer argument '%s' to " + "'%s' test case result") % + argstr % status); + delim = delim2 + 1; + } + + const std::string reason = rest.substr(delim + 2); + + if (status == "expected_exit") + return atf_result(atf_result::expected_exit, arg, reason); + else if (status == "expected_signal") + return atf_result(atf_result::expected_signal, arg, reason); + else + PRE_MSG(false, "Unexpected status"); +} + + +/// Formats the termination status of a process to be used with validate_result. +/// +/// \param status The status to format. +/// +/// \return A string describing the status. +static std::string +format_status(const process::status& status) +{ + if (status.exited()) + return F("exited with code %s") % status.exitstatus(); + else if (status.signaled()) + return F("received signal %s%s") % status.termsig() % + (status.coredump() ? " (core dumped)" : ""); + else + return F("terminated in an unknown manner"); +} + + +} // anonymous namespace + + +/// Constructs a raw result with a type. +/// +/// The reason and the argument are left uninitialized. +/// +/// \param type_ The type of the result. +engine::atf_result::atf_result(const types type_) : + _type(type_) +{ +} + + +/// Constructs a raw result with a type and a reason. +/// +/// The argument is left uninitialized. +/// +/// \param type_ The type of the result. +/// \param reason_ The reason for the result. +engine::atf_result::atf_result(const types type_, const std::string& reason_) : + _type(type_), _reason(reason_) +{ +} + + +/// Constructs a raw result with a type, an optional argument and a reason. +/// +/// \param type_ The type of the result. +/// \param argument_ The optional argument for the result. +/// \param reason_ The reason for the result. +engine::atf_result::atf_result(const types type_, + const utils::optional< int >& argument_, + const std::string& reason_) : + _type(type_), _argument(argument_), _reason(reason_) +{ +} + + +/// Parses an input stream to extract a test result. +/// +/// If the parsing fails for any reason, the test result is 'broken' and it +/// contains the reason for the parsing failure. Test cases that report results +/// in an inconsistent state cannot be trusted (e.g. the test program code may +/// have a bug), and thus why they are reported as broken instead of just failed +/// (which is a legitimate result for a test case). +/// +/// \param input The stream to read from. +/// +/// \return A generic representation of the result of the test case. +/// +/// \throw format_error If the input is invalid. +engine::atf_result +engine::atf_result::parse(std::istream& input) +{ + const std::pair< size_t, std::string > data = read_lines(input); + if (data.first == 0) + throw format_error("Empty test result or no new line"); + else if (data.first > 1) + throw format_error("Test result contains multiple lines: " + + data.second); + else { + const std::string::size_type delim = data.second.find_first_not_of( + "abcdefghijklmnopqrstuvwxyz_"); + const std::string status = data.second.substr(0, delim); + const std::string rest = data.second.substr(status.length()); + + if (status == "broken") + return parse_with_reason(status, rest); + else if (status == "expected_death") + return parse_with_reason(status, rest); + else if (status == "expected_exit") + return parse_with_reason_and_arg(status, rest); + else if (status == "expected_failure") + return parse_with_reason(status, rest); + else if (status == "expected_signal") + return parse_with_reason_and_arg(status, rest); + else if (status == "expected_timeout") + return parse_with_reason(status, rest); + else if (status == "failed") + return parse_with_reason(status, rest); + else if (status == "passed") + return parse_without_reason(status, rest); + else if (status == "skipped") + return parse_with_reason(status, rest); + else + throw format_error(F("Unknown test result '%s'") % status); + } +} + + +/// Loads a test case result from a file. +/// +/// \param file The file to parse. +/// +/// \return The parsed test case result if all goes well. +/// +/// \throw std::runtime_error If the file does not exist. +/// \throw engine::format_error If the contents of the file are bogus. +engine::atf_result +engine::atf_result::load(const fs::path& file) +{ + std::ifstream input(file.c_str()); + if (!input) + throw std::runtime_error("Cannot open results file"); + else + return parse(input); +} + + +/// Gets the type of the result. +/// +/// \return A result type. +engine::atf_result::types +engine::atf_result::type(void) const +{ + return _type; +} + + +/// Gets the optional argument of the result. +/// +/// \return The argument of the result if present; none otherwise. +const optional< int >& +engine::atf_result::argument(void) const +{ + return _argument; +} + + +/// Gets the optional reason of the result. +/// +/// \return The reason of the result if present; none otherwise. +const optional< std::string >& +engine::atf_result::reason(void) const +{ + return _reason; +} + + +/// Checks whether the result should be reported as good or not. +/// +/// \return True if the result can be considered "good", false otherwise. +bool +engine::atf_result::good(void) const +{ + switch (_type) { + case atf_result::expected_death: + case atf_result::expected_exit: + case atf_result::expected_failure: + case atf_result::expected_signal: + case atf_result::expected_timeout: + case atf_result::passed: + case atf_result::skipped: + return true; + + case atf_result::broken: + case atf_result::failed: + return false; + + default: + UNREACHABLE; + } +} + + +/// Reinterprets a raw result based on the termination status of the test case. +/// +/// This reinterpretation ensures that the termination conditions of the program +/// match what is expected of the paticular result reported by the test program. +/// If such conditions do not match, the test program is considered bogus and is +/// thus reported as broken. +/// +/// This is just a helper function for calculate_result(); the real result of +/// the test case cannot be inferred from apply() only. +/// +/// \param status The exit status of the test program, or none if the test +/// program timed out. +/// +/// \result The adjusted result. The original result is transformed into broken +/// if the exit status of the program does not match our expectations. +engine::atf_result +engine::atf_result::apply(const optional< process::status >& status) + const +{ + if (!status) { + if (_type != atf_result::expected_timeout) + return atf_result(atf_result::broken, "Test case body timed out"); + else + return *this; + } + + INV(status); + switch (_type) { + case atf_result::broken: + return *this; + + case atf_result::expected_death: + return *this; + + case atf_result::expected_exit: + if (status.get().exited()) { + if (_argument) { + if (_argument.get() == status.get().exitstatus()) + return *this; + else + return atf_result( + atf_result::failed, + F("Test case expected to exit with code %s but got " + "code %s") % + _argument.get() % status.get().exitstatus()); + } else + return *this; + } else + return atf_result(atf_result::broken, "Expected clean exit but " + + format_status(status.get())); + + case atf_result::expected_failure: + if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS) + return *this; + else + return atf_result(atf_result::broken, "Expected failure should " + "have reported success but " + + format_status(status.get())); + + case atf_result::expected_signal: + if (status.get().signaled()) { + if (_argument) { + if (_argument.get() == status.get().termsig()) + return *this; + else + return atf_result( + atf_result::failed, + F("Test case expected to receive signal %s but " + "got %s") % + _argument.get() % status.get().termsig()); + } else + return *this; + } else + return atf_result(atf_result::broken, "Expected signal but " + + format_status(status.get())); + + case atf_result::expected_timeout: + return atf_result(atf_result::broken, "Expected timeout but " + + format_status(status.get())); + + case atf_result::failed: + if (status.get().exited() && status.get().exitstatus() == EXIT_FAILURE) + return *this; + else + return atf_result(atf_result::broken, "Failed test case should " + "have reported failure but " + + format_status(status.get())); + + case atf_result::passed: + if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS) + return *this; + else + return atf_result(atf_result::broken, "Passed test case should " + "have reported success but " + + format_status(status.get())); + + case atf_result::skipped: + if (status.get().exited() && status.get().exitstatus() == EXIT_SUCCESS) + return *this; + else + return atf_result(atf_result::broken, "Skipped test case should " + "have reported success but " + + format_status(status.get())); + } + + UNREACHABLE; +} + + +/// Converts an internal result to the interface-agnostic representation. +/// +/// \return A generic result instance representing this result. +model::test_result +engine::atf_result::externalize(void) const +{ + switch (_type) { + case atf_result::broken: + return model::test_result(model::test_result_broken, _reason.get()); + + case atf_result::expected_death: + case atf_result::expected_exit: + case atf_result::expected_failure: + case atf_result::expected_signal: + case atf_result::expected_timeout: + return model::test_result(model::test_result_expected_failure, + _reason.get()); + + case atf_result::failed: + return model::test_result(model::test_result_failed, _reason.get()); + + case atf_result::passed: + return model::test_result(model::test_result_passed); + + case atf_result::skipped: + return model::test_result(model::test_result_skipped, _reason.get()); + + default: + UNREACHABLE; + } +} + + +/// Compares two raw results for equality. +/// +/// \param other The result to compare to. +/// +/// \return True if the two raw results are equal; false otherwise. +bool +engine::atf_result::operator==(const atf_result& other) const +{ + return _type == other._type && _argument == other._argument && + _reason == other._reason; +} + + +/// Compares two raw results for inequality. +/// +/// \param other The result to compare to. +/// +/// \return True if the two raw results are different; false otherwise. +bool +engine::atf_result::operator!=(const atf_result& other) const +{ + return !(*this == other); +} + + +/// Injects the object into a stream. +/// +/// \param output The stream into which to inject the object. +/// \param object The object to format. +/// +/// \return The output stream. +std::ostream& +engine::operator<<(std::ostream& output, const atf_result& object) +{ + std::string result_name; + switch (object.type()) { + case atf_result::broken: result_name = "broken"; break; + case atf_result::expected_death: result_name = "expected_death"; break; + case atf_result::expected_exit: result_name = "expected_exit"; break; + case atf_result::expected_failure: result_name = "expected_failure"; break; + case atf_result::expected_signal: result_name = "expected_signal"; break; + case atf_result::expected_timeout: result_name = "expected_timeout"; break; + case atf_result::failed: result_name = "failed"; break; + case atf_result::passed: result_name = "passed"; break; + case atf_result::skipped: result_name = "skipped"; break; + } + + const optional< int >& argument = object.argument(); + + const optional< std::string >& reason = object.reason(); + + output << F("model::test_result{type=%s, argument=%s, reason=%s}") + % text::quote(result_name, '\'') + % (argument ? (F("%s") % argument.get()).str() : "none") + % (reason ? text::quote(reason.get(), '\'') : "none"); + + return output; +} + + +/// Calculates the user-visible result of a test case. +/// +/// This function needs to perform magic to ensure that what the test case +/// reports as its result is what the user should really see: i.e. it adjusts +/// the reported status of the test to the exit conditions of its body and +/// cleanup parts. +/// +/// \param body_status The termination status of the process that executed +/// the body of the test. None if the body timed out. +/// \param results_file The path to the results file that the test case body is +/// supposed to have created. +/// +/// \return The calculated test case result. +model::test_result +engine::calculate_atf_result(const optional< process::status >& body_status, + const fs::path& results_file) +{ + using engine::atf_result; + + atf_result result(atf_result::broken, "Unknown result"); + try { + result = atf_result::load(results_file); + } catch (const engine::format_error& error) { + result = atf_result(atf_result::broken, error.what()); + } catch (const std::runtime_error& error) { + if (body_status) + result = atf_result( + atf_result::broken, F("Premature exit; test case %s") % + format_status(body_status.get())); + else { + // The test case timed out. apply() handles this case later. + } + } + + result = result.apply(body_status); + + return result.externalize(); +} diff --git a/engine/atf_result.hpp b/engine/atf_result.hpp new file mode 100644 index 000000000000..55f8a117a237 --- /dev/null +++ b/engine/atf_result.hpp @@ -0,0 +1,114 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/atf_result.hpp +/// Functions and types to process the results of ATF-based test cases. + +#if !defined(ENGINE_ATF_RESULT_HPP) +#define ENGINE_ATF_RESULT_HPP + +#include "engine/atf_result_fwd.hpp" + +#include <istream> +#include <ostream> + +#include "model/test_result_fwd.hpp" +#include "utils/optional.hpp" +#include "utils/fs/path_fwd.hpp" +#include "utils/process/status_fwd.hpp" + +namespace engine { + + +/// Internal representation of the raw result files of ATF-based tests. +/// +/// This class is used exclusively to represent the transient result files read +/// from test cases before generating the "public" version of the result. This +/// class should actually not be exposed in the header files, but it is for +/// testing purposes only. +class atf_result { +public: + /// List of possible types for the test case result. + enum types { + broken, + expected_death, + expected_exit, + expected_failure, + expected_signal, + expected_timeout, + failed, + passed, + skipped, + }; + +private: + /// The test case result. + types _type; + + /// The optional integral argument that may accompany the result. + /// + /// Should only be present if the type is expected_exit or expected_signal. + utils::optional< int > _argument; + + /// A description of the test case result. + /// + /// Should always be present except for the passed type. + utils::optional< std::string > _reason; + +public: + atf_result(const types); + atf_result(const types, const std::string&); + atf_result(const types, const utils::optional< int >&, const std::string&); + + static atf_result parse(std::istream&); + static atf_result load(const utils::fs::path&); + + types type(void) const; + const utils::optional< int >& argument(void) const; + const utils::optional< std::string >& reason(void) const; + + bool good(void) const; + atf_result apply(const utils::optional< utils::process::status >&) const; + model::test_result externalize(void) const; + + bool operator==(const atf_result&) const; + bool operator!=(const atf_result&) const; +}; + + +std::ostream& operator<<(std::ostream&, const atf_result&); + + +model::test_result calculate_atf_result( + const utils::optional< utils::process::status >&, + const utils::fs::path&); + + +} // namespace engine + +#endif // !defined(ENGINE_ATF_IFACE_RESULTS_HPP) diff --git a/engine/atf_result_fwd.hpp b/engine/atf_result_fwd.hpp new file mode 100644 index 000000000000..2a1440e4929c --- /dev/null +++ b/engine/atf_result_fwd.hpp @@ -0,0 +1,43 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/atf_result_fwd.hpp +/// Forward declarations for engine/atf_result.hpp + +#if !defined(ENGINE_ATF_RESULT_FWD_HPP) +#define ENGINE_ATF_RESULT_FWD_HPP + +namespace engine { + + +class atf_result; + + +} // namespace engine + +#endif // !defined(ENGINE_ATF_RESULT_FWD_HPP) diff --git a/engine/atf_result_test.cpp b/engine/atf_result_test.cpp new file mode 100644 index 000000000000..8ec61dc3c07e --- /dev/null +++ b/engine/atf_result_test.cpp @@ -0,0 +1,788 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/atf_result.hpp" + +extern "C" { +#include <signal.h> +} + +#include <cstdlib> +#include <fstream> +#include <sstream> +#include <stdexcept> + +#include <atf-c++.hpp> + +#include "engine/exceptions.hpp" +#include "model/test_result.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/process/status.hpp" + +namespace fs = utils::fs; +namespace process = utils::process; + +using utils::none; +using utils::optional; + + +namespace { + + +/// Performs a test for results::parse() that should succeed. +/// +/// \param exp_type The expected type of the result. +/// \param exp_argument The expected argument in the result, if any. +/// \param exp_reason The expected reason describing the result, if any. +/// \param text The literal input to parse; can include multiple lines. +static void +parse_ok_test(const engine::atf_result::types& exp_type, + const optional< int >& exp_argument, + const char* exp_reason, const char* text) +{ + std::istringstream input(text); + const engine::atf_result actual = engine::atf_result::parse(input); + ATF_REQUIRE(exp_type == actual.type()); + ATF_REQUIRE_EQ(exp_argument, actual.argument()); + if (exp_reason != NULL) { + ATF_REQUIRE(actual.reason()); + ATF_REQUIRE_EQ(exp_reason, actual.reason().get()); + } else { + ATF_REQUIRE(!actual.reason()); + } +} + + +/// Wrapper around parse_ok_test to define a test case. +/// +/// \param name The name of the test case; will be prefixed with +/// "atf_result__parse__". +/// \param exp_type The expected type of the result. +/// \param exp_argument The expected argument in the result, if any. +/// \param exp_reason The expected reason describing the result, if any. +/// \param input The literal input to parse. +#define PARSE_OK(name, exp_type, exp_argument, exp_reason, input) \ + ATF_TEST_CASE_WITHOUT_HEAD(atf_result__parse__ ## name); \ + ATF_TEST_CASE_BODY(atf_result__parse__ ## name) \ + { \ + parse_ok_test(exp_type, exp_argument, exp_reason, input); \ + } + + +/// Performs a test for results::parse() that should fail. +/// +/// \param reason_regexp The reason to match against the broken reason. +/// \param text The literal input to parse; can include multiple lines. +static void +parse_broken_test(const char* reason_regexp, const char* text) +{ + std::istringstream input(text); + ATF_REQUIRE_THROW_RE(engine::format_error, reason_regexp, + engine::atf_result::parse(input)); +} + + +/// Wrapper around parse_broken_test to define a test case. +/// +/// \param name The name of the test case; will be prefixed with +/// "atf_result__parse__". +/// \param reason_regexp The reason to match against the broken reason. +/// \param input The literal input to parse. +#define PARSE_BROKEN(name, reason_regexp, input) \ + ATF_TEST_CASE_WITHOUT_HEAD(atf_result__parse__ ## name); \ + ATF_TEST_CASE_BODY(atf_result__parse__ ## name) \ + { \ + parse_broken_test(reason_regexp, input); \ + } + + +} // anonymous namespace + + +PARSE_BROKEN(empty, + "Empty.*no new line", + ""); +PARSE_BROKEN(no_newline__unknown, + "Empty.*no new line", + "foo"); +PARSE_BROKEN(no_newline__known, + "Empty.*no new line", + "passed"); +PARSE_BROKEN(multiline__no_newline, + "multiple lines.*foo<<NEWLINE>>bar", + "failed: foo\nbar"); +PARSE_BROKEN(multiline__with_newline, + "multiple lines.*foo<<NEWLINE>>bar", + "failed: foo\nbar\n"); +PARSE_BROKEN(unknown_status__no_reason, + "Unknown.*result.*'cba'", + "cba\n"); +PARSE_BROKEN(unknown_status__with_reason, + "Unknown.*result.*'hgf'", + "hgf: foo\n"); +PARSE_BROKEN(missing_reason__no_delim, + "failed.*followed by.*reason", + "failed\n"); +PARSE_BROKEN(missing_reason__bad_delim, + "failed.*followed by.*reason", + "failed:\n"); +PARSE_BROKEN(missing_reason__empty, + "failed.*followed by.*reason", + "failed: \n"); + + +PARSE_OK(broken__ok, + engine::atf_result::broken, none, "a b c", + "broken: a b c\n"); +PARSE_OK(broken__blanks, + engine::atf_result::broken, none, " ", + "broken: \n"); + + +PARSE_OK(expected_death__ok, + engine::atf_result::expected_death, none, "a b c", + "expected_death: a b c\n"); +PARSE_OK(expected_death__blanks, + engine::atf_result::expected_death, none, " ", + "expected_death: \n"); + + +PARSE_OK(expected_exit__ok__any, + engine::atf_result::expected_exit, none, "any exit code", + "expected_exit: any exit code\n"); +PARSE_OK(expected_exit__ok__specific, + engine::atf_result::expected_exit, optional< int >(712), + "some known exit code", + "expected_exit(712): some known exit code\n"); +PARSE_BROKEN(expected_exit__bad_int, + "Invalid integer.*45a3", + "expected_exit(45a3): this is broken\n"); + + +PARSE_OK(expected_failure__ok, + engine::atf_result::expected_failure, none, "a b c", + "expected_failure: a b c\n"); +PARSE_OK(expected_failure__blanks, + engine::atf_result::expected_failure, none, " ", + "expected_failure: \n"); + + +PARSE_OK(expected_signal__ok__any, + engine::atf_result::expected_signal, none, "any signal code", + "expected_signal: any signal code\n"); +PARSE_OK(expected_signal__ok__specific, + engine::atf_result::expected_signal, optional< int >(712), + "some known signal code", + "expected_signal(712): some known signal code\n"); +PARSE_BROKEN(expected_signal__bad_int, + "Invalid integer.*45a3", + "expected_signal(45a3): this is broken\n"); + + +PARSE_OK(expected_timeout__ok, + engine::atf_result::expected_timeout, none, "a b c", + "expected_timeout: a b c\n"); +PARSE_OK(expected_timeout__blanks, + engine::atf_result::expected_timeout, none, " ", + "expected_timeout: \n"); + + +PARSE_OK(failed__ok, + engine::atf_result::failed, none, "a b c", + "failed: a b c\n"); +PARSE_OK(failed__blanks, + engine::atf_result::failed, none, " ", + "failed: \n"); + + +PARSE_OK(passed__ok, + engine::atf_result::passed, none, NULL, + "passed\n"); +PARSE_BROKEN(passed__reason, + "cannot have a reason", + "passed a b c\n"); + + +PARSE_OK(skipped__ok, + engine::atf_result::skipped, none, "a b c", + "skipped: a b c\n"); +PARSE_OK(skipped__blanks, + engine::atf_result::skipped, none, " ", + "skipped: \n"); + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__ok); +ATF_TEST_CASE_BODY(atf_result__load__ok) +{ + std::ofstream output("result.txt"); + ATF_REQUIRE(output); + output << "skipped: a b c\n"; + output.close(); + + const engine::atf_result result = engine::atf_result::load( + utils::fs::path("result.txt")); + ATF_REQUIRE(engine::atf_result::skipped == result.type()); + ATF_REQUIRE(!result.argument()); + ATF_REQUIRE(result.reason()); + ATF_REQUIRE_EQ("a b c", result.reason().get()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__missing_file); +ATF_TEST_CASE_BODY(atf_result__load__missing_file) +{ + ATF_REQUIRE_THROW_RE( + std::runtime_error, "Cannot open", + engine::atf_result::load(utils::fs::path("result.txt"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__load__format_error); +ATF_TEST_CASE_BODY(atf_result__load__format_error) +{ + std::ofstream output("abc.txt"); + ATF_REQUIRE(output); + output << "passed: foo\n"; + output.close(); + + ATF_REQUIRE_THROW_RE(engine::format_error, "cannot have a reason", + engine::atf_result::load(utils::fs::path("abc.txt"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__broken__ok); +ATF_TEST_CASE_BODY(atf_result__apply__broken__ok) +{ + const engine::atf_result in_result(engine::atf_result::broken, + "Passthrough"); + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + ATF_REQUIRE_EQ(in_result, in_result.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__timed_out); +ATF_TEST_CASE_BODY(atf_result__apply__timed_out) +{ + const engine::atf_result timed_out(engine::atf_result::broken, + "Some arbitrary error"); + ATF_REQUIRE_EQ(engine::atf_result(engine::atf_result::broken, + "Test case body timed out"), + timed_out.apply(none)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_death__ok); +ATF_TEST_CASE_BODY(atf_result__apply__expected_death__ok) +{ + const engine::atf_result in_result(engine::atf_result::expected_death, + "Passthrough"); + const process::status status = process::status::fake_signaled(SIGINT, true); + ATF_REQUIRE_EQ(in_result, in_result.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__ok); +ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__ok) +{ + const process::status success = process::status::fake_exited(EXIT_SUCCESS); + const process::status failure = process::status::fake_exited(EXIT_FAILURE); + + const engine::atf_result any_code(engine::atf_result::expected_exit, none, + "The reason"); + ATF_REQUIRE_EQ(any_code, any_code.apply(utils::make_optional(success))); + ATF_REQUIRE_EQ(any_code, any_code.apply(utils::make_optional(failure))); + + const engine::atf_result a_code(engine::atf_result::expected_exit, + utils::make_optional(EXIT_FAILURE), "The reason"); + ATF_REQUIRE_EQ(a_code, a_code.apply(utils::make_optional(failure))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__failed); +ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__failed) +{ + const process::status success = process::status::fake_exited(EXIT_SUCCESS); + + const engine::atf_result a_code(engine::atf_result::expected_exit, + utils::make_optional(EXIT_FAILURE), "The reason"); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::failed, + "Test case expected to exit with code 1 but got " + "code 0"), + a_code.apply(utils::make_optional(success))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_exit__broken); +ATF_TEST_CASE_BODY(atf_result__apply__expected_exit__broken) +{ + const process::status sig3 = process::status::fake_signaled(3, false); + + const engine::atf_result any_code(engine::atf_result::expected_exit, none, + "The reason"); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Expected clean exit but received signal 3"), + any_code.apply(utils::make_optional(sig3))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_failure__ok); +ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__ok) +{ + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + const engine::atf_result xfailure(engine::atf_result::expected_failure, + "The reason"); + ATF_REQUIRE_EQ(xfailure, xfailure.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_failure__broken); +ATF_TEST_CASE_BODY(atf_result__apply__expected_failure__broken) +{ + const process::status failure = process::status::fake_exited(EXIT_FAILURE); + const process::status sig3 = process::status::fake_signaled(3, true); + const process::status sig4 = process::status::fake_signaled(4, false); + + const engine::atf_result xfailure(engine::atf_result::expected_failure, + "The reason"); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Expected failure should have reported success but " + "exited with code 1"), + xfailure.apply(utils::make_optional(failure))); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Expected failure should have reported success but " + "received signal 3 (core dumped)"), + xfailure.apply(utils::make_optional(sig3))); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Expected failure should have reported success but " + "received signal 4"), + xfailure.apply(utils::make_optional(sig4))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__ok); +ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__ok) +{ + const process::status sig1 = process::status::fake_signaled(1, false); + const process::status sig3 = process::status::fake_signaled(3, true); + + const engine::atf_result any_sig(engine::atf_result::expected_signal, none, + "The reason"); + ATF_REQUIRE_EQ(any_sig, any_sig.apply(utils::make_optional(sig1))); + ATF_REQUIRE_EQ(any_sig, any_sig.apply(utils::make_optional(sig3))); + + const engine::atf_result a_sig(engine::atf_result::expected_signal, + utils::make_optional(3), "The reason"); + ATF_REQUIRE_EQ(a_sig, a_sig.apply(utils::make_optional(sig3))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__failed); +ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__failed) +{ + const process::status sig5 = process::status::fake_signaled(5, false); + + const engine::atf_result a_sig(engine::atf_result::expected_signal, + utils::make_optional(4), "The reason"); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::failed, + "Test case expected to receive signal 4 but got 5"), + a_sig.apply(utils::make_optional(sig5))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_signal__broken); +ATF_TEST_CASE_BODY(atf_result__apply__expected_signal__broken) +{ + const process::status success = process::status::fake_exited(EXIT_SUCCESS); + + const engine::atf_result any_sig(engine::atf_result::expected_signal, none, + "The reason"); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Expected signal but exited with code 0"), + any_sig.apply(utils::make_optional(success))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_timeout__ok); +ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__ok) +{ + const engine::atf_result timeout(engine::atf_result::expected_timeout, + "The reason"); + ATF_REQUIRE_EQ(timeout, timeout.apply(none)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__expected_timeout__broken); +ATF_TEST_CASE_BODY(atf_result__apply__expected_timeout__broken) +{ + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + const engine::atf_result timeout(engine::atf_result::expected_timeout, + "The reason"); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Expected timeout but exited with code 0"), + timeout.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__failed__ok); +ATF_TEST_CASE_BODY(atf_result__apply__failed__ok) +{ + const process::status status = process::status::fake_exited(EXIT_FAILURE); + const engine::atf_result failed(engine::atf_result::failed, "The reason"); + ATF_REQUIRE_EQ(failed, failed.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__failed__broken); +ATF_TEST_CASE_BODY(atf_result__apply__failed__broken) +{ + const process::status success = process::status::fake_exited(EXIT_SUCCESS); + const process::status sig3 = process::status::fake_signaled(3, true); + const process::status sig4 = process::status::fake_signaled(4, false); + + const engine::atf_result failed(engine::atf_result::failed, "The reason"); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Failed test case should have reported failure but " + "exited with code 0"), + failed.apply(utils::make_optional(success))); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Failed test case should have reported failure but " + "received signal 3 (core dumped)"), + failed.apply(utils::make_optional(sig3))); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Failed test case should have reported failure but " + "received signal 4"), + failed.apply(utils::make_optional(sig4))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__passed__ok); +ATF_TEST_CASE_BODY(atf_result__apply__passed__ok) +{ + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + const engine::atf_result passed(engine::atf_result::passed); + ATF_REQUIRE_EQ(passed, passed.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__passed__broken); +ATF_TEST_CASE_BODY(atf_result__apply__passed__broken) +{ + const process::status failure = process::status::fake_exited(EXIT_FAILURE); + const process::status sig3 = process::status::fake_signaled(3, true); + const process::status sig4 = process::status::fake_signaled(4, false); + + const engine::atf_result passed(engine::atf_result::passed); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Passed test case should have reported success but " + "exited with code 1"), + passed.apply(utils::make_optional(failure))); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Passed test case should have reported success but " + "received signal 3 (core dumped)"), + passed.apply(utils::make_optional(sig3))); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Passed test case should have reported success but " + "received signal 4"), + passed.apply(utils::make_optional(sig4))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__skipped__ok); +ATF_TEST_CASE_BODY(atf_result__apply__skipped__ok) +{ + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + const engine::atf_result skipped(engine::atf_result::skipped, "The reason"); + ATF_REQUIRE_EQ(skipped, skipped.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__apply__skipped__broken); +ATF_TEST_CASE_BODY(atf_result__apply__skipped__broken) +{ + const process::status failure = process::status::fake_exited(EXIT_FAILURE); + const process::status sig3 = process::status::fake_signaled(3, true); + const process::status sig4 = process::status::fake_signaled(4, false); + + const engine::atf_result skipped(engine::atf_result::skipped, "The reason"); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Skipped test case should have reported success but " + "exited with code 1"), + skipped.apply(utils::make_optional(failure))); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Skipped test case should have reported success but " + "received signal 3 (core dumped)"), + skipped.apply(utils::make_optional(sig3))); + ATF_REQUIRE_EQ( + engine::atf_result(engine::atf_result::broken, + "Skipped test case should have reported success but " + "received signal 4"), + skipped.apply(utils::make_optional(sig4))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__broken); +ATF_TEST_CASE_BODY(atf_result__externalize__broken) +{ + const engine::atf_result raw(engine::atf_result::broken, "The reason"); + const model::test_result expected(model::test_result_broken, + "The reason"); + ATF_REQUIRE_EQ(expected, raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_death); +ATF_TEST_CASE_BODY(atf_result__externalize__expected_death) +{ + const engine::atf_result raw(engine::atf_result::expected_death, + "The reason"); + const model::test_result expected(model::test_result_expected_failure, + "The reason"); + ATF_REQUIRE_EQ(expected, raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_exit); +ATF_TEST_CASE_BODY(atf_result__externalize__expected_exit) +{ + const engine::atf_result raw(engine::atf_result::expected_exit, + "The reason"); + const model::test_result expected(model::test_result_expected_failure, + "The reason"); + ATF_REQUIRE_EQ(expected, raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_failure); +ATF_TEST_CASE_BODY(atf_result__externalize__expected_failure) +{ + const engine::atf_result raw(engine::atf_result::expected_failure, + "The reason"); + const model::test_result expected(model::test_result_expected_failure, + "The reason"); + ATF_REQUIRE_EQ(expected, raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_signal); +ATF_TEST_CASE_BODY(atf_result__externalize__expected_signal) +{ + const engine::atf_result raw(engine::atf_result::expected_signal, + "The reason"); + const model::test_result expected(model::test_result_expected_failure, + "The reason"); + ATF_REQUIRE_EQ(expected, raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__expected_timeout); +ATF_TEST_CASE_BODY(atf_result__externalize__expected_timeout) +{ + const engine::atf_result raw(engine::atf_result::expected_timeout, + "The reason"); + const model::test_result expected(model::test_result_expected_failure, + "The reason"); + ATF_REQUIRE_EQ(expected, raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__failed); +ATF_TEST_CASE_BODY(atf_result__externalize__failed) +{ + const engine::atf_result raw(engine::atf_result::failed, "The reason"); + const model::test_result expected(model::test_result_failed, + "The reason"); + ATF_REQUIRE(expected == raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__passed); +ATF_TEST_CASE_BODY(atf_result__externalize__passed) +{ + const engine::atf_result raw(engine::atf_result::passed); + const model::test_result expected(model::test_result_passed); + ATF_REQUIRE_EQ(expected, raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(atf_result__externalize__skipped); +ATF_TEST_CASE_BODY(atf_result__externalize__skipped) +{ + const engine::atf_result raw(engine::atf_result::skipped, "The reason"); + const model::test_result expected(model::test_result_skipped, + "The reason"); + ATF_REQUIRE_EQ(expected, raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__missing_file); +ATF_TEST_CASE_BODY(calculate_atf_result__missing_file) +{ + using process::status; + + const status body_status = status::fake_exited(EXIT_SUCCESS); + const model::test_result expected( + model::test_result_broken, + "Premature exit; test case exited with code 0"); + ATF_REQUIRE_EQ(expected, engine::calculate_atf_result( + utils::make_optional(body_status), fs::path("foo"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__bad_file); +ATF_TEST_CASE_BODY(calculate_atf_result__bad_file) +{ + using process::status; + + const status body_status = status::fake_exited(EXIT_SUCCESS); + atf::utils::create_file("foo", "invalid\n"); + const model::test_result expected(model::test_result_broken, + "Unknown test result 'invalid'"); + ATF_REQUIRE_EQ(expected, engine::calculate_atf_result( + utils::make_optional(body_status), fs::path("foo"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__body_ok); +ATF_TEST_CASE_BODY(calculate_atf_result__body_ok) +{ + using process::status; + + atf::utils::create_file("result.txt", "skipped: Something\n"); + const status body_status = status::fake_exited(EXIT_SUCCESS); + ATF_REQUIRE_EQ( + model::test_result(model::test_result_skipped, "Something"), + engine::calculate_atf_result(utils::make_optional(body_status), + fs::path("result.txt"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(calculate_atf_result__body_bad); +ATF_TEST_CASE_BODY(calculate_atf_result__body_bad) +{ + using process::status; + + atf::utils::create_file("result.txt", "skipped: Something\n"); + const status body_status = status::fake_exited(EXIT_FAILURE); + ATF_REQUIRE_EQ( + model::test_result(model::test_result_broken, "Skipped test case " + "should have reported success but exited with " + "code 1"), + engine::calculate_atf_result(utils::make_optional(body_status), + fs::path("result.txt"))); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, atf_result__parse__empty); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__no_newline__unknown); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__no_newline__known); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__multiline__no_newline); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__multiline__with_newline); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__unknown_status__no_reason); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__unknown_status__with_reason); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__no_delim); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__bad_delim); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__missing_reason__empty); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__broken__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__broken__blanks); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_death__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_death__blanks); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__ok__any); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__ok__specific); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_exit__bad_int); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_failure__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_failure__blanks); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__ok__any); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__ok__specific); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_signal__bad_int); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_timeout__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__expected_timeout__blanks); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__failed__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__failed__blanks); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__passed__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__passed__reason); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__skipped__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__parse__skipped__blanks); + + ATF_ADD_TEST_CASE(tcs, atf_result__load__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__load__missing_file); + ATF_ADD_TEST_CASE(tcs, atf_result__load__format_error); + + ATF_ADD_TEST_CASE(tcs, atf_result__apply__broken__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__timed_out); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_death__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__failed); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_exit__broken); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_failure__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_failure__broken); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__failed); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_signal__broken); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_timeout__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__expected_timeout__broken); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__failed__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__failed__broken); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__passed__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__passed__broken); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__skipped__ok); + ATF_ADD_TEST_CASE(tcs, atf_result__apply__skipped__broken); + + ATF_ADD_TEST_CASE(tcs, atf_result__externalize__broken); + ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_death); + ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_exit); + ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_failure); + ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_signal); + ATF_ADD_TEST_CASE(tcs, atf_result__externalize__expected_timeout); + ATF_ADD_TEST_CASE(tcs, atf_result__externalize__failed); + ATF_ADD_TEST_CASE(tcs, atf_result__externalize__passed); + ATF_ADD_TEST_CASE(tcs, atf_result__externalize__skipped); + + ATF_ADD_TEST_CASE(tcs, calculate_atf_result__missing_file); + ATF_ADD_TEST_CASE(tcs, calculate_atf_result__bad_file); + ATF_ADD_TEST_CASE(tcs, calculate_atf_result__body_ok); + ATF_ADD_TEST_CASE(tcs, calculate_atf_result__body_bad); +} diff --git a/engine/atf_test.cpp b/engine/atf_test.cpp new file mode 100644 index 000000000000..9fe7797f4362 --- /dev/null +++ b/engine/atf_test.cpp @@ -0,0 +1,450 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/atf.hpp" + +extern "C" { +#include <sys/stat.h> + +#include <signal.h> +} + +#include <atf-c++.hpp> + +#include "engine/config.hpp" +#include "engine/scheduler.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program_fwd.hpp" +#include "model/test_result.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/env.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/stacktrace.hpp" +#include "utils/test_utils.ipp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace scheduler = engine::scheduler; + +using utils::none; + + +namespace { + + +/// Lists the test cases associated with an ATF test program. +/// +/// \param program_name Basename of the test program to run. +/// \param root Path to the base of the test suite. +/// \param names_filter Whitespace-separated list of test cases that the helper +/// test program is allowed to expose. +/// \param user_config User-provided configuration. +/// +/// \return The list of loaded test cases. +static model::test_cases_map +list_one(const char* program_name, + const fs::path& root, + const char* names_filter = NULL, + config::tree user_config = engine::empty_config()) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + const scheduler::lazy_test_program program( + "atf", fs::path(program_name), root, "the-suite", + model::metadata_builder().build(), user_config, handle); + + if (names_filter != NULL) + utils::setenv("TEST_CASES", names_filter); + const model::test_cases_map test_cases = handle.list_tests( + &program, user_config); + + handle.cleanup(); + + return test_cases; +} + + +/// Runs a bogus test program and checks the error result. +/// +/// \param exp_error Expected error string to find. +/// \param program_name Basename of the test program to run. +/// \param root Path to the base of the test suite. +/// \param names_filter Whitespace-separated list of test cases that the helper +/// test program is allowed to expose. +static void +check_list_one_fail(const char* exp_error, + const char* program_name, + const fs::path& root, + const char* names_filter = NULL) +{ + const model::test_cases_map test_cases = list_one( + program_name, root, names_filter); + + ATF_REQUIRE_EQ(1, test_cases.size()); + const model::test_case& test_case = test_cases.begin()->second; + ATF_REQUIRE_EQ("__test_cases_list__", test_case.name()); + ATF_REQUIRE(test_case.fake_result()); + ATF_REQUIRE_MATCH(exp_error, + test_case.fake_result().get().reason()); +} + + +/// Runs one ATF test program and checks its result. +/// +/// \param tc Pointer to the calling test case, to obtain srcdir. +/// \param test_case_name Name of the "test case" to select from the helper +/// program. +/// \param exp_result The expected result. +/// \param user_config User-provided configuration. +/// \param check_empty_output If true, verify that the output of the test is +/// silent. This is just a hack to implement one of the test cases; we'd +/// easily have a nicer abstraction here... +static void +run_one(const atf::tests::tc* tc, const char* test_case_name, + const model::test_result& exp_result, + config::tree user_config = engine::empty_config(), + const bool check_empty_output = false) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + const model::test_program_ptr program(new scheduler::lazy_test_program( + "atf", fs::path("atf_helpers"), fs::path(tc->get_config_var("srcdir")), + "the-suite", model::metadata_builder().build(), + user_config, handle)); + + (void)handle.spawn_test(program, test_case_name, user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: "); + atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: "); + ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); + if (check_empty_output) { + ATF_REQUIRE(atf::utils::compare_file(result_handle->stdout_file().str(), + "")); + ATF_REQUIRE(atf::utils::compare_file(result_handle->stderr_file().str(), + "")); + } + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(list__ok); +ATF_TEST_CASE_BODY(list__ok) +{ + const model::test_cases_map test_cases = list_one( + "atf_helpers", fs::path(get_config_var("srcdir")), "pass crash"); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("crash") + .add("pass", model::metadata_builder() + .set_description("Always-passing test case") + .build()) + .build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__configuration_variables); +ATF_TEST_CASE_BODY(list__configuration_variables) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.var1", "value1"); + user_config.set_string("test_suites.the-suite.var2", "value2"); + + const model::test_cases_map test_cases = list_one( + "atf_helpers", fs::path(get_config_var("srcdir")), "check_list_config", + user_config); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("check_list_config", model::metadata_builder() + .set_description("Found: var1=value1 var2=value2") + .build()) + .build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__current_directory); +ATF_TEST_CASE_BODY(list__current_directory) +{ + const fs::path helpers = fs::path(get_config_var("srcdir")) / "atf_helpers"; + ATF_REQUIRE(::symlink(helpers.c_str(), "atf_helpers") != -1); + const model::test_cases_map test_cases = list_one( + "atf_helpers", fs::path("."), "pass"); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("pass", model::metadata_builder() + .set_description("Always-passing test case") + .build()) + .build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__relative_path); +ATF_TEST_CASE_BODY(list__relative_path) +{ + const fs::path helpers = fs::path(get_config_var("srcdir")) / "atf_helpers"; + ATF_REQUIRE(::mkdir("dir1", 0755) != -1); + ATF_REQUIRE(::mkdir("dir1/dir2", 0755) != -1); + ATF_REQUIRE(::symlink(helpers.c_str(), "dir1/dir2/atf_helpers") != -1); + const model::test_cases_map test_cases = list_one( + "dir2/atf_helpers", fs::path("dir1"), "pass"); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("pass", model::metadata_builder() + .set_description("Always-passing test case") + .build()) + .build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__missing_test_program); +ATF_TEST_CASE_BODY(list__missing_test_program) +{ + check_list_one_fail("Cannot find test program", "non-existent", + fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__not_a_test_program); +ATF_TEST_CASE_BODY(list__not_a_test_program) +{ + atf::utils::create_file("not-valid", "garbage\n"); + ATF_REQUIRE(::chmod("not-valid", 0755) != -1); + check_list_one_fail("Invalid test program format", "not-valid", + fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__no_permissions); +ATF_TEST_CASE_BODY(list__no_permissions) +{ + atf::utils::create_file("not-executable", "garbage\n"); + check_list_one_fail("Permission denied to run test program", + "not-executable", fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__abort); +ATF_TEST_CASE_BODY(list__abort) +{ + check_list_one_fail("Test program received signal", "atf_helpers", + fs::path(get_config_var("srcdir")), + "crash_head"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__empty); +ATF_TEST_CASE_BODY(list__empty) +{ + check_list_one_fail("No test cases", "atf_helpers", + fs::path(get_config_var("srcdir")), + ""); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__stderr_not_quiet); +ATF_TEST_CASE_BODY(list__stderr_not_quiet) +{ + check_list_one_fail("Test case list wrote to stderr", "atf_helpers", + fs::path(get_config_var("srcdir")), + "output_in_list"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__passes); +ATF_TEST_CASE_BODY(test__body_only__passes) +{ + const model::test_result exp_result(model::test_result_passed); + run_one(this, "pass", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__crashes); +ATF_TEST_CASE_BODY(test__body_only__crashes) +{ + utils::prepare_coredump_test(this); + + const model::test_result exp_result( + model::test_result_broken, + F("Premature exit; test case received signal %s (core dumped)") % + SIGABRT); + run_one(this, "crash", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__times_out); +ATF_TEST_CASE_BODY(test__body_only__times_out) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.control_dir", + fs::current_path().str()); + user_config.set_string("test_suites.the-suite.timeout", "1"); + + const model::test_result exp_result( + model::test_result_broken, "Test case body timed out"); + run_one(this, "timeout_body", exp_result, user_config); + + ATF_REQUIRE(!atf::utils::file_exists("cookie")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__configuration_variables); +ATF_TEST_CASE_BODY(test__body_only__configuration_variables) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.first", "some value"); + user_config.set_string("test_suites.the-suite.second", "some other value"); + + const model::test_result exp_result(model::test_result_passed); + run_one(this, "check_configuration_variables", exp_result, user_config); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__no_atf_run_warning); +ATF_TEST_CASE_BODY(test__body_only__no_atf_run_warning) +{ + const model::test_result exp_result(model::test_result_passed); + run_one(this, "pass", exp_result, engine::empty_config(), true); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__body_times_out); +ATF_TEST_CASE_BODY(test__body_and_cleanup__body_times_out) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.control_dir", + fs::current_path().str()); + user_config.set_string("test_suites.the-suite.timeout", "1"); + + const model::test_result exp_result( + model::test_result_broken, "Test case body timed out"); + run_one(this, "timeout_body", exp_result, user_config); + + ATF_REQUIRE(!atf::utils::file_exists("cookie")); + ATF_REQUIRE(atf::utils::file_exists("cookie.cleanup")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__cleanup_crashes); +ATF_TEST_CASE_BODY(test__body_and_cleanup__cleanup_crashes) +{ + const model::test_result exp_result( + model::test_result_broken, + "Test case cleanup did not terminate successfully"); + run_one(this, "crash_cleanup", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__cleanup_times_out); +ATF_TEST_CASE_BODY(test__body_and_cleanup__cleanup_times_out) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.control_dir", + fs::current_path().str()); + + scheduler::cleanup_timeout = datetime::delta(1, 0); + const model::test_result exp_result( + model::test_result_broken, "Test case cleanup timed out"); + run_one(this, "timeout_cleanup", exp_result, user_config); + + ATF_REQUIRE(!atf::utils::file_exists("cookie")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__expect_timeout); +ATF_TEST_CASE_BODY(test__body_and_cleanup__expect_timeout) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.control_dir", + fs::current_path().str()); + user_config.set_string("test_suites.the-suite.timeout", "1"); + + const model::test_result exp_result( + model::test_result_expected_failure, "Times out on purpose"); + run_one(this, "expect_timeout", exp_result, user_config); + + ATF_REQUIRE(!atf::utils::file_exists("cookie")); + ATF_REQUIRE(atf::utils::file_exists("cookie.cleanup")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_and_cleanup__shared_workdir); +ATF_TEST_CASE_BODY(test__body_and_cleanup__shared_workdir) +{ + const model::test_result exp_result(model::test_result_passed); + run_one(this, "shared_workdir", exp_result); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + scheduler::register_interface( + "atf", std::shared_ptr< scheduler::interface >( + new engine::atf_interface())); + + ATF_ADD_TEST_CASE(tcs, list__ok); + ATF_ADD_TEST_CASE(tcs, list__configuration_variables); + ATF_ADD_TEST_CASE(tcs, list__current_directory); + ATF_ADD_TEST_CASE(tcs, list__relative_path); + ATF_ADD_TEST_CASE(tcs, list__missing_test_program); + ATF_ADD_TEST_CASE(tcs, list__not_a_test_program); + ATF_ADD_TEST_CASE(tcs, list__no_permissions); + ATF_ADD_TEST_CASE(tcs, list__abort); + ATF_ADD_TEST_CASE(tcs, list__empty); + ATF_ADD_TEST_CASE(tcs, list__stderr_not_quiet); + + ATF_ADD_TEST_CASE(tcs, test__body_only__passes); + ATF_ADD_TEST_CASE(tcs, test__body_only__crashes); + ATF_ADD_TEST_CASE(tcs, test__body_only__times_out); + ATF_ADD_TEST_CASE(tcs, test__body_only__configuration_variables); + ATF_ADD_TEST_CASE(tcs, test__body_only__no_atf_run_warning); + ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__body_times_out); + ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__cleanup_crashes); + ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__cleanup_times_out); + ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__expect_timeout); + ATF_ADD_TEST_CASE(tcs, test__body_and_cleanup__shared_workdir); +} diff --git a/engine/config.cpp b/engine/config.cpp new file mode 100644 index 000000000000..3f162a94fbb5 --- /dev/null +++ b/engine/config.cpp @@ -0,0 +1,254 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/config.hpp" + +#if defined(HAVE_CONFIG_H) +# include "config.h" +#endif + +#include <stdexcept> + +#include "engine/exceptions.hpp" +#include "utils/config/exceptions.hpp" +#include "utils/config/parser.hpp" +#include "utils/config/tree.ipp" +#include "utils/passwd.hpp" +#include "utils/text/exceptions.hpp" +#include "utils/text/operations.ipp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace passwd = utils::passwd; +namespace text = utils::text; + + +namespace { + + +/// Defines the schema of a configuration tree. +/// +/// \param [in,out] tree The tree to populate. The tree should be empty on +/// entry to prevent collisions with the keys defined in here. +static void +init_tree(config::tree& tree) +{ + tree.define< config::string_node >("architecture"); + tree.define< config::positive_int_node >("parallelism"); + tree.define< config::string_node >("platform"); + tree.define< engine::user_node >("unprivileged_user"); + tree.define_dynamic("test_suites"); +} + + +/// Fills in a configuration tree with default values. +/// +/// \param [in,out] tree The tree to populate. init_tree() must have been +/// called on it beforehand. +static void +set_defaults(config::tree& tree) +{ + tree.set< config::string_node >("architecture", KYUA_ARCHITECTURE); + // TODO(jmmv): Automatically derive this from the number of CPUs in the + // machine and forcibly set to a value greater than 1. Still testing + // the new parallel implementation as of 2015-02-27 though. + tree.set< config::positive_int_node >("parallelism", 1); + tree.set< config::string_node >("platform", KYUA_PLATFORM); +} + + +/// Configuration parser specialization for Kyua configuration files. +class config_parser : public config::parser { + /// Initializes the configuration tree. + /// + /// This is a callback executed when the configuration script invokes the + /// syntax() method. We populate the configuration tree from here with the + /// schema version requested by the file. + /// + /// \param [in,out] tree The tree to populate. + /// \param syntax_version The version of the file format as specified in the + /// configuration file. + /// + /// \throw config::syntax_error If the syntax_format/syntax_version + /// combination is not supported. + void + setup(config::tree& tree, const int syntax_version) + { + if (syntax_version < 1 || syntax_version > 2) + throw config::syntax_error(F("Unsupported config version %s") % + syntax_version); + + init_tree(tree); + set_defaults(tree); + } + +public: + /// Initializes the parser. + /// + /// \param [out] tree_ The tree in which the results of the parsing will be + /// stored when parse() is called. Should be empty on entry. Because + /// we grab a reference to this object, the tree must remain valid for + /// the existence of the parser object. + explicit config_parser(config::tree& tree_) : + config::parser(tree_) + { + } +}; + + +} // anonymous namespace + + +/// Copies the node. +/// +/// \return A dynamically-allocated node. +config::detail::base_node* +engine::user_node::deep_copy(void) const +{ + std::auto_ptr< user_node > new_node(new user_node()); + new_node->_value = _value; + return new_node.release(); +} + + +/// Pushes the node's value onto the Lua stack. +/// +/// \param state The Lua state onto which to push the value. +void +engine::user_node::push_lua(lutok::state& state) const +{ + state.push_string(value().name); +} + + +/// Sets the value of the node from an entry in the Lua stack. +/// +/// \param state The Lua state from which to get the value. +/// \param value_index The stack index in which the value resides. +/// +/// \throw value_error If the value in state(value_index) cannot be +/// processed by this node. +void +engine::user_node::set_lua(lutok::state& state, const int value_index) +{ + if (state.is_number(value_index)) { + config::typed_leaf_node< passwd::user >::set( + passwd::find_user_by_uid(state.to_integer(-1))); + } else if (state.is_string(value_index)) { + config::typed_leaf_node< passwd::user >::set( + passwd::find_user_by_name(state.to_string(-1))); + } else + throw config::value_error("Invalid user identifier"); +} + + +/// Sets the value of the node from a raw string representation. +/// +/// \param raw_value The value to set the node to. +/// +/// \throw value_error If the value is invalid. +void +engine::user_node::set_string(const std::string& raw_value) +{ + try { + config::typed_leaf_node< passwd::user >::set( + passwd::find_user_by_name(raw_value)); + } catch (const std::runtime_error& e) { + int uid; + try { + uid = text::to_type< int >(raw_value); + } catch (const text::value_error& e2) { + throw error(F("Cannot find user with name '%s'") % raw_value); + } + + try { + config::typed_leaf_node< passwd::user >::set( + passwd::find_user_by_uid(uid)); + } catch (const std::runtime_error& e2) { + throw error(F("Cannot find user with UID %s") % uid); + } + } +} + + +/// Converts the contents of the node to a string. +/// +/// \pre The node must have a value. +/// +/// \return A string representation of the value held by the node. +std::string +engine::user_node::to_string(void) const +{ + return config::typed_leaf_node< passwd::user >::value().name; +} + + +/// Constructs a config with the built-in settings. +/// +/// \return A default test suite configuration. +config::tree +engine::default_config(void) +{ + config::tree tree(false); + init_tree(tree); + set_defaults(tree); + return tree; +} + + +/// Constructs a config with the built-in settings. +/// +/// \return An empty test suite configuration. +config::tree +engine::empty_config(void) +{ + config::tree tree(false); + init_tree(tree); + return tree; +} + + +/// Parses a test suite configuration file. +/// +/// \param file The file to parse. +/// +/// \return High-level representation of the configuration file. +/// +/// \throw load_error If there is any problem loading the file. This includes +/// file access errors and syntax errors. +config::tree +engine::load_config(const utils::fs::path& file) +{ + config::tree tree(false); + try { + config_parser(tree).parse(file); + } catch (const config::error& e) { + throw load_error(file, e.what()); + } + return tree; +} diff --git a/engine/config.hpp b/engine/config.hpp new file mode 100644 index 000000000000..2c1b83481862 --- /dev/null +++ b/engine/config.hpp @@ -0,0 +1,65 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/config.hpp +/// Test suite configuration parsing and representation. + +#if !defined(ENGINE_CONFIG_HPP) +#define ENGINE_CONFIG_HPP + +#include "engine/config_fwd.hpp" + +#include "utils/config/nodes.hpp" +#include "utils/config/tree_fwd.hpp" +#include "utils/fs/path_fwd.hpp" +#include "utils/passwd_fwd.hpp" + +namespace engine { + + +/// Tree node to hold a system user identifier. +class user_node : public utils::config::typed_leaf_node< utils::passwd::user > { +public: + virtual base_node* deep_copy(void) const; + + void push_lua(lutok::state&) const; + void set_lua(lutok::state&, const int); + + void set_string(const std::string&); + std::string to_string(void) const; +}; + + +utils::config::tree default_config(void); +utils::config::tree empty_config(void); +utils::config::tree load_config(const utils::fs::path&); + + +} // namespace engine + +#endif // !defined(ENGINE_CONFIG_HPP) diff --git a/engine/config_fwd.hpp b/engine/config_fwd.hpp new file mode 100644 index 000000000000..82da9b1382bd --- /dev/null +++ b/engine/config_fwd.hpp @@ -0,0 +1,43 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/config_fwd.hpp +/// Forward declarations for engine/config.hpp + +#if !defined(ENGINE_CONFIG_FWD_HPP) +#define ENGINE_CONFIG_FWD_HPP + +namespace engine { + + +class user_node; + + +} // namespace engine + +#endif // !defined(ENGINE_CONFIG_FWD_HPP) diff --git a/engine/config_test.cpp b/engine/config_test.cpp new file mode 100644 index 000000000000..e4eb27421078 --- /dev/null +++ b/engine/config_test.cpp @@ -0,0 +1,203 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/config.hpp" + +#if defined(HAVE_CONFIG_H) +# include "config.h" +#endif + +#include <stdexcept> +#include <vector> + +#include <atf-c++.hpp> + +#include "engine/exceptions.hpp" +#include "utils/cmdline/exceptions.hpp" +#include "utils/cmdline/parser.hpp" +#include "utils/config/tree.ipp" +#include "utils/passwd.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace passwd = utils::passwd; + +using utils::none; +using utils::optional; + + +namespace { + + +/// Replaces the system user database with a fake one for testing purposes. +static void +set_mock_users(void) +{ + std::vector< passwd::user > users; + users.push_back(passwd::user("user1", 100, 150)); + users.push_back(passwd::user("user2", 200, 250)); + passwd::set_mock_users_for_testing(users); +} + + +/// Checks that the default values of a config object match our expectations. +/// +/// This fails the test case if any field of the input config object is not +/// what we expect. +/// +/// \param config The configuration to validate. +static void +validate_defaults(const config::tree& config) +{ + ATF_REQUIRE_EQ( + KYUA_ARCHITECTURE, + config.lookup< config::string_node >("architecture")); + + ATF_REQUIRE_EQ( + 1, + config.lookup< config::positive_int_node >("parallelism")); + + ATF_REQUIRE_EQ( + KYUA_PLATFORM, + config.lookup< config::string_node >("platform")); + + ATF_REQUIRE(!config.is_set("unprivileged_user")); + + ATF_REQUIRE(config.all_properties("test_suites").empty()); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(config__defaults); +ATF_TEST_CASE_BODY(config__defaults) +{ + const config::tree user_config = engine::default_config(); + validate_defaults(user_config); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(config__set__parallelism); +ATF_TEST_CASE_BODY(config__set__parallelism) +{ + config::tree user_config = engine::default_config(); + user_config.set_string("parallelism", "8"); + ATF_REQUIRE_THROW_RE( + config::error, "parallelism.*Must be a positive integer", + user_config.set_string("parallelism", "0")); + ATF_REQUIRE_THROW_RE( + config::error, "parallelism.*Must be a positive integer", + user_config.set_string("parallelism", "-1")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(config__load__defaults); +ATF_TEST_CASE_BODY(config__load__defaults) +{ + atf::utils::create_file("config", "syntax(2)\n"); + + const config::tree user_config = engine::load_config(fs::path("config")); + validate_defaults(user_config); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(config__load__overrides); +ATF_TEST_CASE_BODY(config__load__overrides) +{ + set_mock_users(); + + atf::utils::create_file( + "config", + "syntax(2)\n" + "architecture = 'test-architecture'\n" + "parallelism = 16\n" + "platform = 'test-platform'\n" + "unprivileged_user = 'user2'\n" + "test_suites.mysuite.myvar = 'myvalue'\n"); + + const config::tree user_config = engine::load_config(fs::path("config")); + + ATF_REQUIRE_EQ("test-architecture", + user_config.lookup_string("architecture")); + ATF_REQUIRE_EQ("16", + user_config.lookup_string("parallelism")); + ATF_REQUIRE_EQ("test-platform", + user_config.lookup_string("platform")); + + const passwd::user& user = user_config.lookup< engine::user_node >( + "unprivileged_user"); + ATF_REQUIRE_EQ("user2", user.name); + ATF_REQUIRE_EQ(200, user.uid); + + config::properties_map exp_test_suites; + exp_test_suites["test_suites.mysuite.myvar"] = "myvalue"; + + ATF_REQUIRE(exp_test_suites == user_config.all_properties("test_suites")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(config__load__lua_error); +ATF_TEST_CASE_BODY(config__load__lua_error) +{ + atf::utils::create_file("config", "this syntax is invalid\n"); + + ATF_REQUIRE_THROW(engine::load_error, engine::load_config( + fs::path("config"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(config__load__bad_syntax__version); +ATF_TEST_CASE_BODY(config__load__bad_syntax__version) +{ + atf::utils::create_file("config", "syntax(123)\n"); + + ATF_REQUIRE_THROW_RE(engine::load_error, + "Unsupported config version 123", + engine::load_config(fs::path("config"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(config__load__missing_file); +ATF_TEST_CASE_BODY(config__load__missing_file) +{ + ATF_REQUIRE_THROW_RE(engine::load_error, "Load of 'missing' failed", + engine::load_config(fs::path("missing"))); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, config__defaults); + ATF_ADD_TEST_CASE(tcs, config__set__parallelism); + ATF_ADD_TEST_CASE(tcs, config__load__defaults); + ATF_ADD_TEST_CASE(tcs, config__load__overrides); + ATF_ADD_TEST_CASE(tcs, config__load__lua_error); + ATF_ADD_TEST_CASE(tcs, config__load__bad_syntax__version); + ATF_ADD_TEST_CASE(tcs, config__load__missing_file); +} diff --git a/engine/exceptions.cpp b/engine/exceptions.cpp new file mode 100644 index 000000000000..98a7b43a7de3 --- /dev/null +++ b/engine/exceptions.cpp @@ -0,0 +1,81 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/exceptions.hpp" + +#include "utils/format/macros.hpp" + +namespace fs = utils::fs; + +/// Constructs a new error with a plain-text message. +/// +/// \param message The plain-text error message. +engine::error::error(const std::string& message) : + std::runtime_error(message) +{ +} + + +/// Destructor for the error. +engine::error::~error(void) throw() +{ +} + + +/// Constructs a new format_error. +/// +/// \param reason_ Description of the format problem. +engine::format_error::format_error(const std::string& reason_) : + error(reason_) +{ +} + + +/// Destructor for the error. +engine::format_error::~format_error(void) throw() +{ +} + + +/// Constructs a new load_error. +/// +/// \param file_ The file in which the error was encountered. +/// \param reason_ Description of the load problem. +engine::load_error::load_error(const fs::path& file_, + const std::string& reason_) : + error(F("Load of '%s' failed: %s") % file_ % reason_), + file(file_), + reason(reason_) +{ +} + + +/// Destructor for the error. +engine::load_error::~load_error(void) throw() +{ +} diff --git a/engine/exceptions.hpp b/engine/exceptions.hpp new file mode 100644 index 000000000000..fccb04f1aff2 --- /dev/null +++ b/engine/exceptions.hpp @@ -0,0 +1,75 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/exceptions.hpp +/// Exception types raised by the engine module. + +#if !defined(ENGINE_EXCEPTIONS_HPP) +#define ENGINE_EXCEPTIONS_HPP + +#include <stdexcept> + +#include "utils/fs/path.hpp" + +namespace engine { + + +/// Base exception for engine errors. +class error : public std::runtime_error { +public: + explicit error(const std::string&); + virtual ~error(void) throw(); +}; + + +/// Error while processing data. +class format_error : public error { +public: + explicit format_error(const std::string&); + virtual ~format_error(void) throw(); +}; + + +/// Error while parsing external data. +class load_error : public error { +public: + /// The path to the file that caused the load error. + utils::fs::path file; + + /// The reason for the error; may not include the file name. + std::string reason; + + explicit load_error(const utils::fs::path&, const std::string&); + virtual ~load_error(void) throw(); +}; + + +} // namespace engine + + +#endif // !defined(ENGINE_EXCEPTIONS_HPP) diff --git a/engine/exceptions_test.cpp b/engine/exceptions_test.cpp new file mode 100644 index 000000000000..16e7c9f33d16 --- /dev/null +++ b/engine/exceptions_test.cpp @@ -0,0 +1,69 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/exceptions.hpp" + +#include <cstring> + +#include <atf-c++.hpp> + +namespace fs = utils::fs; + + +ATF_TEST_CASE_WITHOUT_HEAD(error); +ATF_TEST_CASE_BODY(error) +{ + const engine::error e("Some text"); + ATF_REQUIRE(std::strcmp("Some text", e.what()) == 0); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(format_error); +ATF_TEST_CASE_BODY(format_error) +{ + const engine::format_error e("Some text"); + ATF_REQUIRE(std::strcmp("Some text", e.what()) == 0); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(load_error); +ATF_TEST_CASE_BODY(load_error) +{ + const engine::load_error e(fs::path("/my/file"), "foo"); + ATF_REQUIRE_EQ(fs::path("/my/file"), e.file); + ATF_REQUIRE_EQ("foo", e.reason); + ATF_REQUIRE(std::strcmp("Load of '/my/file' failed: foo", e.what()) == 0); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, error); + ATF_ADD_TEST_CASE(tcs, format_error); + ATF_ADD_TEST_CASE(tcs, load_error); +} diff --git a/engine/filters.cpp b/engine/filters.cpp new file mode 100644 index 000000000000..753e64ae05f8 --- /dev/null +++ b/engine/filters.cpp @@ -0,0 +1,389 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/filters.hpp" + +#include <algorithm> +#include <stdexcept> + +#include "utils/format/macros.hpp" +#include "utils/fs/exceptions.hpp" +#include "utils/logging/macros.hpp" +#include "utils/optional.ipp" +#include "utils/sanity.hpp" + +namespace fs = utils::fs; + +using utils::none; +using utils::optional; + + +/// Constructs a filter. +/// +/// \param test_program_ The name of the test program or of the subdirectory to +/// match. +/// \param test_case_ The name of the test case to match. +engine::test_filter::test_filter(const fs::path& test_program_, + const std::string& test_case_) : + test_program(test_program_), + test_case(test_case_) +{ +} + + +/// Parses a user-provided test filter. +/// +/// \param str The user-provided string representing a filter for tests. Must +/// be of the form <test_program%gt;[:<test_case%gt;]. +/// +/// \return The parsed filter. +/// +/// \throw std::runtime_error If the provided filter is invalid. +engine::test_filter +engine::test_filter::parse(const std::string& str) +{ + if (str.empty()) + throw std::runtime_error("Test filter cannot be empty"); + + const std::string::size_type pos = str.find(':'); + if (pos == 0) + throw std::runtime_error(F("Program name component in '%s' is empty") + % str); + if (pos == str.length() - 1) + throw std::runtime_error(F("Test case component in '%s' is empty") + % str); + + try { + const fs::path test_program_(str.substr(0, pos)); + if (test_program_.is_absolute()) + throw std::runtime_error(F("Program name '%s' must be relative " + "to the test suite, not absolute") % + test_program_.str()); + if (pos == std::string::npos) { + LD(F("Parsed user filter '%s': test program '%s', no test case") % + str % test_program_.str()); + return test_filter(test_program_, ""); + } else { + const std::string test_case_(str.substr(pos + 1)); + LD(F("Parsed user filter '%s': test program '%s', test case '%s'") % + str % test_program_.str() % test_case_); + return test_filter(test_program_, test_case_); + } + } catch (const fs::error& e) { + throw std::runtime_error(F("Invalid path in filter '%s': %s") % str % + e.what()); + } +} + + +/// Formats a filter for user presentation. +/// +/// \return A user-friendly string representing the filter. Note that this does +/// not necessarily match the string the user provided: in particular, the path +/// may have been internally normalized. +std::string +engine::test_filter::str(void) const +{ + if (!test_case.empty()) + return F("%s:%s") % test_program % test_case; + else + return test_program.str(); +} + + +/// Checks if this filter contains another. +/// +/// \param other The filter to compare to. +/// +/// \return True if this filter contains the other filter or if they are equal. +bool +engine::test_filter::contains(const test_filter& other) const +{ + if (*this == other) + return true; + else + return test_case.empty() && test_program.is_parent_of( + other.test_program); +} + + +/// Checks if this filter matches a given test program name or subdirectory. +/// +/// \param test_program_ The test program to compare to. +/// +/// \return Whether the filter matches the test program. This is a superset of +/// matches_test_case. +bool +engine::test_filter::matches_test_program(const fs::path& test_program_) const +{ + if (test_program == test_program_) + return true; + else { + // Check if the filter matches a subdirectory of the test program. + // The test case must be empty because we don't want foo:bar to match + // foo/baz. + return (test_case.empty() && test_program.is_parent_of(test_program_)); + } +} + + +/// Checks if this filter matches a given test case identifier. +/// +/// \param test_program_ The test program to compare to. +/// \param test_case_ The test case to compare to. +/// +/// \return Whether the filter matches the test case. +bool +engine::test_filter::matches_test_case(const fs::path& test_program_, + const std::string& test_case_) const +{ + if (matches_test_program(test_program_)) { + return test_case.empty() || test_case == test_case_; + } else + return false; +} + + +/// Less-than comparison for sorting purposes. +/// +/// \param other The filter to compare to. +/// +/// \return True if this filter sorts before the other filter. +bool +engine::test_filter::operator<(const test_filter& other) const +{ + return ( + test_program < other.test_program || + (test_program == other.test_program && test_case < other.test_case)); +} + + +/// Equality comparison. +/// +/// \param other The filter to compare to. +/// +/// \return True if this filter is equal to the other filter. +bool +engine::test_filter::operator==(const test_filter& other) const +{ + return test_program == other.test_program && test_case == other.test_case; +} + + +/// Non-equality comparison. +/// +/// \param other The filter to compare to. +/// +/// \return True if this filter is different than the other filter. +bool +engine::test_filter::operator!=(const test_filter& other) const +{ + return !(*this == other); +} + + +/// Injects the object into a stream. +/// +/// \param output The stream into which to inject the object. +/// \param object The object to format. +/// +/// \return The output stream. +std::ostream& +engine::operator<<(std::ostream& output, const test_filter& object) +{ + if (object.test_case.empty()) { + output << F("test_filter{test_program=%s}") % object.test_program; + } else { + output << F("test_filter{test_program=%s, test_case=%s}") + % object.test_program % object.test_case; + } + return output; +} + + +/// Constructs a new set of filters. +/// +/// \param filters_ The filters themselves; if empty, no filters are applied. +engine::test_filters::test_filters(const std::set< test_filter >& filters_) : + _filters(filters_) +{ +} + + +/// Checks if a given test program matches the set of filters. +/// +/// This is provided as an optimization only, and the results of this function +/// are less specific than those of match_test_case. Checking for the matching +/// of a test program should be done before loading the list of test cases from +/// a program, so as to avoid the delay in executing the test program, but +/// match_test_case must still be called afterwards. +/// +/// \param name The test program to check against the filters. +/// +/// \return True if the provided identifier matches any filter. +bool +engine::test_filters::match_test_program(const fs::path& name) const +{ + if (_filters.empty()) + return true; + + bool matches = false; + for (std::set< test_filter >::const_iterator iter = _filters.begin(); + !matches && iter != _filters.end(); iter++) { + matches = (*iter).matches_test_program(name); + } + return matches; +} + + +/// Checks if a given test case identifier matches the set of filters. +/// +/// \param test_program The test program to check against the filters. +/// \param test_case The test case to check against the filters. +/// +/// \return A boolean indicating if the test case is matched by any filter and, +/// if true, a string containing the filter name. The string is empty when +/// there are no filters defined. +engine::test_filters::match +engine::test_filters::match_test_case(const fs::path& test_program, + const std::string& test_case) const +{ + if (_filters.empty()) { + INV(match_test_program(test_program)); + return match(true, none); + } + + optional< test_filter > found = none; + for (std::set< test_filter >::const_iterator iter = _filters.begin(); + !found && iter != _filters.end(); iter++) { + if ((*iter).matches_test_case(test_program, test_case)) + found = *iter; + } + INV(!found || match_test_program(test_program)); + return match(static_cast< bool >(found), found); +} + + +/// Calculates the filters that have not matched any tests. +/// +/// \param matched The filters that did match some tests. This must be a subset +/// of the filters held by this object. +/// +/// \return The set of filters that have not been used. +std::set< engine::test_filter > +engine::test_filters::difference(const std::set< test_filter >& matched) const +{ + PRE(std::includes(_filters.begin(), _filters.end(), + matched.begin(), matched.end())); + + std::set< test_filter > filters; + std::set_difference(_filters.begin(), _filters.end(), + matched.begin(), matched.end(), + std::inserter(filters, filters.begin())); + return filters; +} + + +/// Checks if a collection of filters is disjoint. +/// +/// \param filters The filters to check. +/// +/// \throw std::runtime_error If the filters are not disjoint. +void +engine::check_disjoint_filters(const std::set< engine::test_filter >& filters) +{ + // Yes, this is an O(n^2) algorithm. However, we can assume that the number + // of test filters (which are provided by the user on the command line) on a + // particular run is in the order of tens, and thus this should not cause + // any serious performance trouble. + for (std::set< test_filter >::const_iterator i1 = filters.begin(); + i1 != filters.end(); i1++) { + for (std::set< test_filter >::const_iterator i2 = filters.begin(); + i2 != filters.end(); i2++) { + const test_filter& filter1 = *i1; + const test_filter& filter2 = *i2; + + if (i1 != i2 && filter1.contains(filter2)) { + throw std::runtime_error( + F("Filters '%s' and '%s' are not disjoint") % + filter1.str() % filter2.str()); + } + } + } +} + + +/// Constructs a filters_state instance. +/// +/// \param filters_ The set of filters to track. +engine::filters_state::filters_state( + const std::set< engine::test_filter >& filters_) : + _filters(test_filters(filters_)) +{ +} + + +/// Checks whether these filters match the given test program. +/// +/// \param test_program The test program to match against. +/// +/// \return True if these filters match the given test program name. +bool +engine::filters_state::match_test_program(const fs::path& test_program) const +{ + return _filters.match_test_program(test_program); +} + + +/// Checks whether these filters match the given test case. +/// +/// \param test_program The test program to match against. +/// \param test_case The test case to match against. +/// +/// \return True if these filters match the given test case identifier. +bool +engine::filters_state::match_test_case(const fs::path& test_program, + const std::string& test_case) +{ + engine::test_filters::match match = _filters.match_test_case( + test_program, test_case); + if (match.first && match.second) + _used_filters.insert(match.second.get()); + return match.first; +} + + +/// Calculates the unused filters in this set. +/// +/// \return Returns the set of filters that have not matched any tests. This +/// information is useful to report usage errors to the user. +std::set< engine::test_filter > +engine::filters_state::unused(void) const +{ + return _filters.difference(_used_filters); +} diff --git a/engine/filters.hpp b/engine/filters.hpp new file mode 100644 index 000000000000..91a667c3b46b --- /dev/null +++ b/engine/filters.hpp @@ -0,0 +1,134 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/filters.hpp +/// Representation and manipulation of filters for test cases. +/// +/// All the filter classes in this module are supposed to be purely functional: +/// they are mere filters that decide whether they match or not the input data +/// fed to them. User-interface filter manipulation must go somewhere else. + +#if !defined(ENGINE_FILTERS_HPP) +#define ENGINE_FILTERS_HPP + +#include "engine/filters_fwd.hpp" + +#include <ostream> +#include <string> +#include <set> +#include <utility> + +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" + + +namespace engine { + + +/// Filter for test cases. +/// +/// A filter is one of: the name of a directory containing test cases, the name +/// of a test program, or the name of a test program plus the name of a test +/// case. +class test_filter { +public: + /// The name of the test program or subdirectory to match. + utils::fs::path test_program; + + /// The name of the test case to match; if empty, represents any test case. + std::string test_case; + + test_filter(const utils::fs::path&, const std::string&); + static test_filter parse(const std::string&); + + std::string str(void) const; + + bool contains(const test_filter&) const; + bool matches_test_program(const utils::fs::path&) const; + bool matches_test_case(const utils::fs::path&, const std::string&) const; + + bool operator<(const test_filter&) const; + bool operator==(const test_filter&) const; + bool operator!=(const test_filter&) const; +}; + + +std::ostream& operator<<(std::ostream&, const test_filter&); + + +/// Collection of user-provided filters to select test cases. +/// +/// An empty collection of filters is considered to match any test case. +/// +/// In general, the filters maintained by this class should be disjoint. If +/// they are not, some filters may never have a chance to do a match, which is +/// most likely the fault of the user. To check for non-disjoint filters before +/// constructing this object, use check_disjoint_filters. +class test_filters { + /// The user-provided filters. + std::set< test_filter > _filters; + +public: + explicit test_filters(const std::set< test_filter >&); + + /// Return type of match_test_case. Indicates whether the filters have + /// matched a particular test case and, if they have, which filter did the + /// match (if any). + typedef std::pair< bool, utils::optional< test_filter > > match; + + bool match_test_program(const utils::fs::path&) const; + match match_test_case(const utils::fs::path&, const std::string&) const; + + std::set< test_filter > difference(const std::set< test_filter >&) const; +}; + + +void check_disjoint_filters(const std::set< test_filter >&); + + +/// Tracks state of the filters that have matched tests during execution. +class filters_state { + /// The user-provided filters. + test_filters _filters; + + /// Collection of filters that have matched test cases so far. + std::set< test_filter > _used_filters; + +public: + explicit filters_state(const std::set< test_filter >&); + + bool match_test_program(const utils::fs::path&) const; + bool match_test_case(const utils::fs::path&, const std::string&); + + std::set< test_filter > unused(void) const; +}; + + +} // namespace engine + +#endif // !defined(ENGINE_FILTERS_HPP) diff --git a/engine/filters_fwd.hpp b/engine/filters_fwd.hpp new file mode 100644 index 000000000000..ee5d0c692ff5 --- /dev/null +++ b/engine/filters_fwd.hpp @@ -0,0 +1,45 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/filters_fwd.hpp +/// Forward declarations for engine/filters.hpp + +#if !defined(ENGINE_FILTERS_FWD_HPP) +#define ENGINE_FILTERS_FWD_HPP + +namespace engine { + + +class filters_state; +class test_filter; +class test_filters; + + +} // namespace engine + +#endif // !defined(ENGINE_FILTERS_FWD_HPP) diff --git a/engine/filters_test.cpp b/engine/filters_test.cpp new file mode 100644 index 000000000000..081755b2553f --- /dev/null +++ b/engine/filters_test.cpp @@ -0,0 +1,594 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/filters.hpp" + +#include <stdexcept> + +#include <atf-c++.hpp> + +namespace fs = utils::fs; + + +namespace { + + +/// Syntactic sugar to instantiate engine::test_filter objects. +/// +/// \param test_program Test program. +/// \param test_case Test case. +/// +/// \return A \p test_filter object, based on \p test_program and \p test_case. +inline engine::test_filter +mkfilter(const char* test_program, const char* test_case) +{ + return engine::test_filter(fs::path(test_program), test_case); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__public_fields); +ATF_TEST_CASE_BODY(test_filter__public_fields) +{ + const engine::test_filter filter(fs::path("foo/bar"), "baz"); + ATF_REQUIRE_EQ(fs::path("foo/bar"), filter.test_program); + ATF_REQUIRE_EQ("baz", filter.test_case); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__ok); +ATF_TEST_CASE_BODY(test_filter__parse__ok) +{ + const engine::test_filter filter(engine::test_filter::parse("foo")); + ATF_REQUIRE_EQ(fs::path("foo"), filter.test_program); + ATF_REQUIRE(filter.test_case.empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__empty); +ATF_TEST_CASE_BODY(test_filter__parse__empty) +{ + ATF_REQUIRE_THROW_RE(std::runtime_error, "empty", + engine::test_filter::parse("")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__absolute); +ATF_TEST_CASE_BODY(test_filter__parse__absolute) +{ + ATF_REQUIRE_THROW_RE(std::runtime_error, "'/foo/bar'.*relative", + engine::test_filter::parse("/foo//bar")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__bad_program_name); +ATF_TEST_CASE_BODY(test_filter__parse__bad_program_name) +{ + ATF_REQUIRE_THROW_RE(std::runtime_error, "Program name.*':foo'", + engine::test_filter::parse(":foo")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__bad_test_case); +ATF_TEST_CASE_BODY(test_filter__parse__bad_test_case) +{ + ATF_REQUIRE_THROW_RE(std::runtime_error, "Test case.*'bar/baz:'", + engine::test_filter::parse("bar/baz:")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__parse__bad_path); +ATF_TEST_CASE_BODY(test_filter__parse__bad_path) +{ + // TODO(jmmv): Not implemented. At the moment, the only reason for a path + // to be invalid is if it is empty... but we are checking this exact + // condition ourselves as part of the input validation. So we can't mock in + // an argument with an invalid non-empty path... +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__str); +ATF_TEST_CASE_BODY(test_filter__str) +{ + const engine::test_filter filter(fs::path("foo/bar"), "baz"); + ATF_REQUIRE_EQ("foo/bar:baz", filter.str()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__contains__same); +ATF_TEST_CASE_BODY(test_filter__contains__same) +{ + { + const engine::test_filter f(fs::path("foo/bar"), "baz"); + ATF_REQUIRE(f.contains(f)); + } + { + const engine::test_filter f(fs::path("foo/bar"), ""); + ATF_REQUIRE(f.contains(f)); + } + { + const engine::test_filter f(fs::path("foo"), ""); + ATF_REQUIRE(f.contains(f)); + } +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__contains__different); +ATF_TEST_CASE_BODY(test_filter__contains__different) +{ + { + const engine::test_filter f1(fs::path("foo"), ""); + const engine::test_filter f2(fs::path("foo"), "bar"); + ATF_REQUIRE( f1.contains(f2)); + ATF_REQUIRE(!f2.contains(f1)); + } + { + const engine::test_filter f1(fs::path("foo/bar"), ""); + const engine::test_filter f2(fs::path("foo/bar"), "baz"); + ATF_REQUIRE( f1.contains(f2)); + ATF_REQUIRE(!f2.contains(f1)); + } + { + const engine::test_filter f1(fs::path("foo/bar"), ""); + const engine::test_filter f2(fs::path("foo/baz"), ""); + ATF_REQUIRE(!f1.contains(f2)); + ATF_REQUIRE(!f2.contains(f1)); + } + { + const engine::test_filter f1(fs::path("foo"), ""); + const engine::test_filter f2(fs::path("foo/bar"), ""); + ATF_REQUIRE( f1.contains(f2)); + ATF_REQUIRE(!f2.contains(f1)); + } + { + const engine::test_filter f1(fs::path("foo"), "bar"); + const engine::test_filter f2(fs::path("foo/bar"), ""); + ATF_REQUIRE(!f1.contains(f2)); + ATF_REQUIRE(!f2.contains(f1)); + } +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__matches_test_program) +ATF_TEST_CASE_BODY(test_filter__matches_test_program) +{ + { + const engine::test_filter f(fs::path("top"), "unused"); + ATF_REQUIRE( f.matches_test_program(fs::path("top"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("top2"))); + } + + { + const engine::test_filter f(fs::path("dir1/dir2"), ""); + ATF_REQUIRE( f.matches_test_program(fs::path("dir1/dir2/foo"))); + ATF_REQUIRE( f.matches_test_program(fs::path("dir1/dir2/bar"))); + ATF_REQUIRE( f.matches_test_program(fs::path("dir1/dir2/bar/baz"))); + ATF_REQUIRE( f.matches_test_program(fs::path("dir1/dir2/bar/baz"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("dir1"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/bar/baz"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("dir2/bar/baz"))); + } + + { + const engine::test_filter f(fs::path("dir1/dir2"), "unused"); + ATF_REQUIRE( f.matches_test_program(fs::path("dir1/dir2"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/dir2/foo"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/dir2/bar"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/dir2/bar/baz"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/dir2/bar/baz"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("dir1"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("dir1/bar/baz"))); + ATF_REQUIRE(!f.matches_test_program(fs::path("dir2/bar/baz"))); + } +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__matches_test_case) +ATF_TEST_CASE_BODY(test_filter__matches_test_case) +{ + { + const engine::test_filter f(fs::path("top"), "foo"); + ATF_REQUIRE( f.matches_test_case(fs::path("top"), "foo")); + ATF_REQUIRE(!f.matches_test_case(fs::path("top"), "bar")); + } + + { + const engine::test_filter f(fs::path("top"), ""); + ATF_REQUIRE( f.matches_test_case(fs::path("top"), "foo")); + ATF_REQUIRE( f.matches_test_case(fs::path("top"), "bar")); + ATF_REQUIRE(!f.matches_test_case(fs::path("top2"), "foo")); + } + + { + const engine::test_filter f(fs::path("d1/d2/prog"), "t1"); + ATF_REQUIRE( f.matches_test_case(fs::path("d1/d2/prog"), "t1")); + ATF_REQUIRE(!f.matches_test_case(fs::path("d1/d2/prog"), "t2")); + } + + { + const engine::test_filter f(fs::path("d1/d2"), ""); + ATF_REQUIRE( f.matches_test_case(fs::path("d1/d2/prog"), "t1")); + ATF_REQUIRE( f.matches_test_case(fs::path("d1/d2/prog"), "t2")); + ATF_REQUIRE( f.matches_test_case(fs::path("d1/d2/prog2"), "t2")); + ATF_REQUIRE(!f.matches_test_case(fs::path("d1/d3"), "foo")); + ATF_REQUIRE(!f.matches_test_case(fs::path("d2"), "foo")); + } +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__operator_lt) +ATF_TEST_CASE_BODY(test_filter__operator_lt) +{ + { + const engine::test_filter f1(fs::path("d1/d2"), ""); + ATF_REQUIRE(!(f1 < f1)); + } + { + const engine::test_filter f1(fs::path("d1/d2"), ""); + const engine::test_filter f2(fs::path("d1/d3"), ""); + ATF_REQUIRE( (f1 < f2)); + ATF_REQUIRE(!(f2 < f1)); + } + { + const engine::test_filter f1(fs::path("d1/d2"), ""); + const engine::test_filter f2(fs::path("d1/d2"), "foo"); + ATF_REQUIRE( (f1 < f2)); + ATF_REQUIRE(!(f2 < f1)); + } + { + const engine::test_filter f1(fs::path("d1/d2"), "bar"); + const engine::test_filter f2(fs::path("d1/d2"), "foo"); + ATF_REQUIRE( (f1 < f2)); + ATF_REQUIRE(!(f2 < f1)); + } + { + const engine::test_filter f1(fs::path("d1/d2"), "bar"); + const engine::test_filter f2(fs::path("d1/d3"), ""); + ATF_REQUIRE( (f1 < f2)); + ATF_REQUIRE(!(f2 < f1)); + } +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__operator_eq) +ATF_TEST_CASE_BODY(test_filter__operator_eq) +{ + const engine::test_filter f1(fs::path("d1/d2"), ""); + const engine::test_filter f2(fs::path("d1/d2"), "bar"); + ATF_REQUIRE( (f1 == f1)); + ATF_REQUIRE(!(f1 == f2)); + ATF_REQUIRE(!(f2 == f1)); + ATF_REQUIRE( (f2 == f2)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__operator_ne) +ATF_TEST_CASE_BODY(test_filter__operator_ne) +{ + const engine::test_filter f1(fs::path("d1/d2"), ""); + const engine::test_filter f2(fs::path("d1/d2"), "bar"); + ATF_REQUIRE(!(f1 != f1)); + ATF_REQUIRE( (f1 != f2)); + ATF_REQUIRE( (f2 != f1)); + ATF_REQUIRE(!(f2 != f2)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filter__output); +ATF_TEST_CASE_BODY(test_filter__output) +{ + { + std::ostringstream str; + str << engine::test_filter(fs::path("d1/d2"), ""); + ATF_REQUIRE_EQ( + "test_filter{test_program=d1/d2}", + str.str()); + } + { + std::ostringstream str; + str << engine::test_filter(fs::path("d1/d2"), "bar"); + ATF_REQUIRE_EQ( + "test_filter{test_program=d1/d2, test_case=bar}", + str.str()); + } +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filters__match_test_case__no_filters) +ATF_TEST_CASE_BODY(test_filters__match_test_case__no_filters) +{ + const std::set< engine::test_filter > raw_filters; + + const engine::test_filters filters(raw_filters); + engine::test_filters::match match; + + match = filters.match_test_case(fs::path("foo"), "baz"); + ATF_REQUIRE(match.first); + ATF_REQUIRE(!match.second); + + match = filters.match_test_case(fs::path("foo/bar"), "baz"); + ATF_REQUIRE(match.first); + ATF_REQUIRE(!match.second); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filters__match_test_case__some_filters) +ATF_TEST_CASE_BODY(test_filters__match_test_case__some_filters) +{ + std::set< engine::test_filter > raw_filters; + raw_filters.insert(mkfilter("top_test", "")); + raw_filters.insert(mkfilter("subdir_1", "")); + raw_filters.insert(mkfilter("subdir_2/a_test", "")); + raw_filters.insert(mkfilter("subdir_2/b_test", "foo")); + + const engine::test_filters filters(raw_filters); + engine::test_filters::match match; + + match = filters.match_test_case(fs::path("top_test"), "a"); + ATF_REQUIRE(match.first); + ATF_REQUIRE_EQ("top_test", match.second.get().str()); + + match = filters.match_test_case(fs::path("subdir_1/foo"), "a"); + ATF_REQUIRE(match.first); + ATF_REQUIRE_EQ("subdir_1", match.second.get().str()); + + match = filters.match_test_case(fs::path("subdir_1/bar"), "z"); + ATF_REQUIRE(match.first); + ATF_REQUIRE_EQ("subdir_1", match.second.get().str()); + + match = filters.match_test_case(fs::path("subdir_2/a_test"), "bar"); + ATF_REQUIRE(match.first); + ATF_REQUIRE_EQ("subdir_2/a_test", match.second.get().str()); + + match = filters.match_test_case(fs::path("subdir_2/b_test"), "foo"); + ATF_REQUIRE(match.first); + ATF_REQUIRE_EQ("subdir_2/b_test:foo", match.second.get().str()); + + match = filters.match_test_case(fs::path("subdir_2/b_test"), "bar"); + ATF_REQUIRE(!match.first); + + match = filters.match_test_case(fs::path("subdir_2/c_test"), "foo"); + ATF_REQUIRE(!match.first); + + match = filters.match_test_case(fs::path("subdir_3"), "hello"); + ATF_REQUIRE(!match.first); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filters__match_test_program__no_filters) +ATF_TEST_CASE_BODY(test_filters__match_test_program__no_filters) +{ + const std::set< engine::test_filter > raw_filters; + + const engine::test_filters filters(raw_filters); + ATF_REQUIRE(filters.match_test_program(fs::path("foo"))); + ATF_REQUIRE(filters.match_test_program(fs::path("foo/bar"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filters__match_test_program__some_filters) +ATF_TEST_CASE_BODY(test_filters__match_test_program__some_filters) +{ + std::set< engine::test_filter > raw_filters; + raw_filters.insert(mkfilter("top_test", "")); + raw_filters.insert(mkfilter("subdir_1", "")); + raw_filters.insert(mkfilter("subdir_2/a_test", "")); + raw_filters.insert(mkfilter("subdir_2/b_test", "foo")); + + const engine::test_filters filters(raw_filters); + ATF_REQUIRE( filters.match_test_program(fs::path("top_test"))); + ATF_REQUIRE( filters.match_test_program(fs::path("subdir_1/foo"))); + ATF_REQUIRE( filters.match_test_program(fs::path("subdir_1/bar"))); + ATF_REQUIRE( filters.match_test_program(fs::path("subdir_2/a_test"))); + ATF_REQUIRE( filters.match_test_program(fs::path("subdir_2/b_test"))); + ATF_REQUIRE(!filters.match_test_program(fs::path("subdir_2/c_test"))); + ATF_REQUIRE(!filters.match_test_program(fs::path("subdir_3"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filters__difference__no_filters); +ATF_TEST_CASE_BODY(test_filters__difference__no_filters) +{ + const std::set< engine::test_filter > in_filters; + const std::set< engine::test_filter > used; + const std::set< engine::test_filter > diff = engine::test_filters( + in_filters).difference(used); + ATF_REQUIRE(diff.empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filters__difference__some_filters__all_used); +ATF_TEST_CASE_BODY(test_filters__difference__some_filters__all_used) +{ + std::set< engine::test_filter > in_filters; + in_filters.insert(mkfilter("a", "")); + in_filters.insert(mkfilter("b", "c")); + + const std::set< engine::test_filter > used = in_filters; + + const std::set< engine::test_filter > diff = engine::test_filters( + in_filters).difference(used); + ATF_REQUIRE(diff.empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test_filters__difference__some_filters__some_unused); +ATF_TEST_CASE_BODY(test_filters__difference__some_filters__some_unused) +{ + std::set< engine::test_filter > in_filters; + in_filters.insert(mkfilter("a", "")); + in_filters.insert(mkfilter("b", "c")); + in_filters.insert(mkfilter("d", "")); + in_filters.insert(mkfilter("e", "f")); + + std::set< engine::test_filter > used; + used.insert(mkfilter("b", "c")); + used.insert(mkfilter("d", "")); + + const std::set< engine::test_filter > diff = engine::test_filters( + in_filters).difference(used); + ATF_REQUIRE_EQ(2, diff.size()); + ATF_REQUIRE(diff.find(mkfilter("a", "")) != diff.end()); + ATF_REQUIRE(diff.find(mkfilter("e", "f")) != diff.end()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_disjoint_filters__ok); +ATF_TEST_CASE_BODY(check_disjoint_filters__ok) +{ + std::set< engine::test_filter > filters; + filters.insert(mkfilter("a", "")); + filters.insert(mkfilter("b", "")); + filters.insert(mkfilter("c", "a")); + filters.insert(mkfilter("c", "b")); + + engine::check_disjoint_filters(filters); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_disjoint_filters__fail); +ATF_TEST_CASE_BODY(check_disjoint_filters__fail) +{ + std::set< engine::test_filter > filters; + filters.insert(mkfilter("a", "")); + filters.insert(mkfilter("b", "")); + filters.insert(mkfilter("c", "a")); + filters.insert(mkfilter("d", "b")); + filters.insert(mkfilter("c", "")); + + ATF_REQUIRE_THROW_RE(std::runtime_error, "'c'.*'c:a'.*not disjoint", + engine::check_disjoint_filters(filters)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(filters_state__match_test_program); +ATF_TEST_CASE_BODY(filters_state__match_test_program) +{ + std::set< engine::test_filter > filters; + filters.insert(mkfilter("foo/bar", "")); + filters.insert(mkfilter("baz", "tc")); + engine::filters_state state(filters); + + ATF_REQUIRE(state.match_test_program(fs::path("foo/bar/something"))); + ATF_REQUIRE(state.match_test_program(fs::path("baz"))); + + ATF_REQUIRE(!state.match_test_program(fs::path("foo/baz"))); + ATF_REQUIRE(!state.match_test_program(fs::path("hello"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(filters_state__match_test_case); +ATF_TEST_CASE_BODY(filters_state__match_test_case) +{ + std::set< engine::test_filter > filters; + filters.insert(mkfilter("foo/bar", "")); + filters.insert(mkfilter("baz", "tc")); + engine::filters_state state(filters); + + ATF_REQUIRE(state.match_test_case(fs::path("foo/bar/something"), "any")); + ATF_REQUIRE(state.match_test_case(fs::path("baz"), "tc")); + + ATF_REQUIRE(!state.match_test_case(fs::path("foo/baz/something"), "tc")); + ATF_REQUIRE(!state.match_test_case(fs::path("baz"), "tc2")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(filters_state__unused__none); +ATF_TEST_CASE_BODY(filters_state__unused__none) +{ + std::set< engine::test_filter > filters; + filters.insert(mkfilter("a/b", "")); + filters.insert(mkfilter("baz", "tc")); + filters.insert(mkfilter("hey/d", "yes")); + engine::filters_state state(filters); + + state.match_test_case(fs::path("a/b/c"), "any"); + state.match_test_case(fs::path("baz"), "tc"); + state.match_test_case(fs::path("hey/d"), "yes"); + + ATF_REQUIRE(state.unused().empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(filters_state__unused__some); +ATF_TEST_CASE_BODY(filters_state__unused__some) +{ + std::set< engine::test_filter > filters; + filters.insert(mkfilter("a/b", "")); + filters.insert(mkfilter("baz", "tc")); + filters.insert(mkfilter("hey/d", "yes")); + engine::filters_state state(filters); + + state.match_test_program(fs::path("a/b/c")); + state.match_test_case(fs::path("baz"), "tc"); + + std::set< engine::test_filter > exp_unused; + exp_unused.insert(mkfilter("a/b", "")); + exp_unused.insert(mkfilter("hey/d", "yes")); + + ATF_REQUIRE(exp_unused == state.unused()); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, test_filter__public_fields); + ATF_ADD_TEST_CASE(tcs, test_filter__parse__ok); + ATF_ADD_TEST_CASE(tcs, test_filter__parse__empty); + ATF_ADD_TEST_CASE(tcs, test_filter__parse__absolute); + ATF_ADD_TEST_CASE(tcs, test_filter__parse__bad_program_name); + ATF_ADD_TEST_CASE(tcs, test_filter__parse__bad_test_case); + ATF_ADD_TEST_CASE(tcs, test_filter__parse__bad_path); + ATF_ADD_TEST_CASE(tcs, test_filter__str); + ATF_ADD_TEST_CASE(tcs, test_filter__contains__same); + ATF_ADD_TEST_CASE(tcs, test_filter__contains__different); + ATF_ADD_TEST_CASE(tcs, test_filter__matches_test_program); + ATF_ADD_TEST_CASE(tcs, test_filter__matches_test_case); + ATF_ADD_TEST_CASE(tcs, test_filter__operator_lt); + ATF_ADD_TEST_CASE(tcs, test_filter__operator_eq); + ATF_ADD_TEST_CASE(tcs, test_filter__operator_ne); + ATF_ADD_TEST_CASE(tcs, test_filter__output); + + ATF_ADD_TEST_CASE(tcs, test_filters__match_test_case__no_filters); + ATF_ADD_TEST_CASE(tcs, test_filters__match_test_case__some_filters); + ATF_ADD_TEST_CASE(tcs, test_filters__match_test_program__no_filters); + ATF_ADD_TEST_CASE(tcs, test_filters__match_test_program__some_filters); + ATF_ADD_TEST_CASE(tcs, test_filters__difference__no_filters); + ATF_ADD_TEST_CASE(tcs, test_filters__difference__some_filters__all_used); + ATF_ADD_TEST_CASE(tcs, test_filters__difference__some_filters__some_unused); + + ATF_ADD_TEST_CASE(tcs, check_disjoint_filters__ok); + ATF_ADD_TEST_CASE(tcs, check_disjoint_filters__fail); + + ATF_ADD_TEST_CASE(tcs, filters_state__match_test_program); + ATF_ADD_TEST_CASE(tcs, filters_state__match_test_case); + ATF_ADD_TEST_CASE(tcs, filters_state__unused__none); + ATF_ADD_TEST_CASE(tcs, filters_state__unused__some); +} diff --git a/engine/kyuafile.cpp b/engine/kyuafile.cpp new file mode 100644 index 000000000000..4dca3193832b --- /dev/null +++ b/engine/kyuafile.cpp @@ -0,0 +1,694 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/kyuafile.hpp" + +#include <algorithm> +#include <iterator> +#include <stdexcept> + +#include <lutok/exceptions.hpp> +#include <lutok/operations.hpp> +#include <lutok/stack_cleaner.hpp> +#include <lutok/state.ipp> + +#include "engine/exceptions.hpp" +#include "engine/scheduler.hpp" +#include "model/metadata.hpp" +#include "model/test_program.hpp" +#include "utils/config/exceptions.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/lua_module.hpp" +#include "utils/fs/operations.hpp" +#include "utils/logging/macros.hpp" +#include "utils/noncopyable.hpp" +#include "utils/optional.ipp" +#include "utils/sanity.hpp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace scheduler = engine::scheduler; + +using utils::none; +using utils::optional; + + +// History of Kyuafile file versions: +// +// 3 - DOES NOT YET EXIST. Pending changes for when this is introduced: +// +// * Revisit what to do about the test_suite definition. Support for +// per-test program overrides is deprecated and should be removed. +// But, maybe, the whole test_suite definition idea is wrong and we +// should instead be explicitly telling which configuration variables +// to "inject" into each test program. +// +// 2 - Changed the syntax() call to take only a version number, instead of the +// word 'config' as the first argument and the version as the second one. +// Files now start with syntax(2) instead of syntax('kyuafile', 1). +// +// 1 - Initial version. + + +namespace { + + +static int lua_current_kyuafile(lutok::state&); +static int lua_generic_test_program(lutok::state&); +static int lua_include(lutok::state&); +static int lua_syntax(lutok::state&); +static int lua_test_suite(lutok::state&); + + +/// Concatenates two paths while avoiding paths to start with './'. +/// +/// \param root Path to the directory containing the file. +/// \param file Path to concatenate to root. Cannot be absolute. +/// +/// \return The concatenated path. +static fs::path +relativize(const fs::path& root, const fs::path& file) +{ + PRE(!file.is_absolute()); + + if (root == fs::path(".")) + return file; + else + return root / file; +} + + +/// Implementation of a parser for Kyuafiles. +/// +/// The main purpose of having this as a class is to keep track of global state +/// within the Lua files and allowing the Lua callbacks to easily access such +/// data. +class parser : utils::noncopyable { + /// Lua state to parse a single Kyuafile file. + lutok::state _state; + + /// Root directory of the test suite represented by the Kyuafile. + const fs::path _source_root; + + /// Root directory of the test programs. + const fs::path _build_root; + + /// Name of the Kyuafile to load relative to _source_root. + const fs::path _relative_filename; + + /// Version of the Kyuafile file format requested by the parsed file. + /// + /// This is set once the Kyuafile invokes the syntax() call. + optional< int > _version; + + /// Name of the test suite defined by the Kyuafile. + /// + /// This is set once the Kyuafile invokes the test_suite() call. + optional< std::string > _test_suite; + + /// Collection of test programs defined by the Kyuafile. + /// + /// This acts as an accumulator for all the *_test_program() calls within + /// the Kyuafile. + model::test_programs_vector _test_programs; + + /// Safely gets _test_suite and respects any test program overrides. + /// + /// \param program_override The test program-specific test suite name. May + /// be empty to indicate no override. + /// + /// \return The name of the test suite. + /// + /// \throw std::runtime_error If program_override is empty and the Kyuafile + /// did not yet define the global name of the test suite. + std::string + get_test_suite(const std::string& program_override) + { + std::string test_suite; + + if (program_override.empty()) { + if (!_test_suite) { + throw std::runtime_error("No test suite defined in the " + "Kyuafile and no override provided in " + "the test_program definition"); + } + test_suite = _test_suite.get(); + } else { + test_suite = program_override; + } + + return test_suite; + } + +public: + /// Initializes the parser and the Lua state. + /// + /// \param source_root_ The root directory of the test suite represented by + /// the Kyuafile. + /// \param build_root_ The root directory of the test programs. + /// \param relative_filename_ Name of the Kyuafile to load relative to + /// source_root_. + /// \param user_config User configuration holding any test suite properties + /// to be passed to the list operation. + /// \param scheduler_handle The scheduler context to use for loading the + /// test case lists. + parser(const fs::path& source_root_, const fs::path& build_root_, + const fs::path& relative_filename_, + const config::tree& user_config, + scheduler::scheduler_handle& scheduler_handle) : + _source_root(source_root_), _build_root(build_root_), + _relative_filename(relative_filename_) + { + lutok::stack_cleaner cleaner(_state); + + _state.push_cxx_function(lua_syntax); + _state.set_global("syntax"); + + *_state.new_userdata< parser* >() = this; + _state.set_global("_parser"); + + _state.push_cxx_function(lua_current_kyuafile); + _state.set_global("current_kyuafile"); + + *_state.new_userdata< const config::tree* >() = &user_config; + *_state.new_userdata< scheduler::scheduler_handle* >() = + &scheduler_handle; + _state.push_cxx_closure(lua_include, 2); + _state.set_global("include"); + + _state.push_cxx_function(lua_test_suite); + _state.set_global("test_suite"); + + const std::set< std::string > interfaces = + scheduler::registered_interface_names(); + for (std::set< std::string >::const_iterator iter = interfaces.begin(); + iter != interfaces.end(); ++iter) { + const std::string& interface = *iter; + + _state.push_string(interface); + *_state.new_userdata< const config::tree* >() = &user_config; + *_state.new_userdata< scheduler::scheduler_handle* >() = + &scheduler_handle; + _state.push_cxx_closure(lua_generic_test_program, 3); + _state.set_global(interface + "_test_program"); + } + + _state.open_base(); + _state.open_string(); + _state.open_table(); + fs::open_fs(_state, callback_current_kyuafile().branch_path()); + } + + /// Destructor. + ~parser(void) + { + } + + /// Gets the parser object associated to a Lua state. + /// + /// \param state The Lua state from which to obtain the parser object. + /// + /// \return A pointer to the parser. + static parser* + get_from_state(lutok::state& state) + { + lutok::stack_cleaner cleaner(state); + state.get_global("_parser"); + return *state.to_userdata< parser* >(-1); + } + + /// Callback for the Kyuafile current_kyuafile() function. + /// + /// \return Returns the absolute path to the current Kyuafile. + fs::path + callback_current_kyuafile(void) const + { + const fs::path file = relativize(_source_root, _relative_filename); + if (file.is_absolute()) + return file; + else + return file.to_absolute(); + } + + /// Callback for the Kyuafile include() function. + /// + /// \post _test_programs is extended with the the test programs defined by + /// the included file. + /// + /// \param raw_file Path to the file to include. + /// \param user_config User configuration holding any test suite properties + /// to be passed to the list operation. + /// \param scheduler_handle Scheduler context to run test programs in. + void + callback_include(const fs::path& raw_file, + const config::tree& user_config, + scheduler::scheduler_handle& scheduler_handle) + { + const fs::path file = relativize(_relative_filename.branch_path(), + raw_file); + const model::test_programs_vector subtps = + parser(_source_root, _build_root, file, user_config, + scheduler_handle).parse(); + + std::copy(subtps.begin(), subtps.end(), + std::back_inserter(_test_programs)); + } + + /// Callback for the Kyuafile syntax() function. + /// + /// \post _version is set to the requested version. + /// + /// \param version Version of the Kyuafile syntax requested by the file. + /// + /// \throw std::runtime_error If the format or the version are invalid, or + /// if syntax() has already been called. + void + callback_syntax(const int version) + { + if (_version) + throw std::runtime_error("Can only call syntax() once"); + + if (version < 1 || version > 2) + throw std::runtime_error(F("Unsupported file version %s") % + version); + + _version = utils::make_optional(version); + } + + /// Callback for the various Kyuafile *_test_program() functions. + /// + /// \post _test_programs is extended to include the newly defined test + /// program. + /// + /// \param interface Name of the test program interface. + /// \param raw_path Path to the test program, relative to the Kyuafile. + /// This has to be adjusted according to the relative location of this + /// Kyuafile to _source_root. + /// \param test_suite_override Name of the test suite this test program + /// belongs to, if explicitly defined at the test program level. + /// \param metadata Metadata variables passed to the test program. + /// \param user_config User configuration holding any test suite properties + /// to be passed to the list operation. + /// \param scheduler_handle Scheduler context to run test programs in. + /// + /// \throw std::runtime_error If the test program definition is invalid or + /// if the test program does not exist. + void + callback_test_program(const std::string& interface, + const fs::path& raw_path, + const std::string& test_suite_override, + const model::metadata& metadata, + const config::tree& user_config, + scheduler::scheduler_handle& scheduler_handle) + { + if (raw_path.is_absolute()) + throw std::runtime_error(F("Got unexpected absolute path for test " + "program '%s'") % raw_path); + else if (raw_path.str() != raw_path.leaf_name()) + throw std::runtime_error(F("Test program '%s' cannot contain path " + "components") % raw_path); + + const fs::path path = relativize(_relative_filename.branch_path(), + raw_path); + + if (!fs::exists(_build_root / path)) + throw std::runtime_error(F("Non-existent test program '%s'") % + path); + + const std::string test_suite = get_test_suite(test_suite_override); + + _test_programs.push_back(model::test_program_ptr( + new scheduler::lazy_test_program(interface, path, _build_root, + test_suite, metadata, user_config, + scheduler_handle))); + } + + /// Callback for the Kyuafile test_suite() function. + /// + /// \post _version is set to the requested version. + /// + /// \param name Name of the test suite. + /// + /// \throw std::runtime_error If test_suite() has already been called. + void + callback_test_suite(const std::string& name) + { + if (_test_suite) + throw std::runtime_error("Can only call test_suite() once"); + _test_suite = utils::make_optional(name); + } + + /// Parses the Kyuafile. + /// + /// \pre Can only be invoked once. + /// + /// \return The collection of test programs defined by the Kyuafile. + /// + /// \throw load_error If there is any problem parsing the file. + const model::test_programs_vector& + parse(void) + { + PRE(_test_programs.empty()); + + const fs::path load_path = relativize(_source_root, _relative_filename); + try { + lutok::do_file(_state, load_path.str(), 0, 0, 0); + } catch (const std::runtime_error& e) { + // It is tempting to think that all of our various auxiliary + // functions above could raise load_error by themselves thus making + // this exception rewriting here unnecessary. Howver, that would + // not work because the helper functions above are executed within a + // Lua context, and we lose their type when they are propagated out + // of it. + throw engine::load_error(load_path, e.what()); + } + + if (!_version) + throw engine::load_error(load_path, "syntax() never called"); + + return _test_programs; + } +}; + + +/// Glue to invoke parser::callback_test_program() from Lua. +/// +/// This is a helper function for the various *_test_program() calls, as they +/// only differ in the interface of the defined test program. +/// +/// \pre state(-1) A table with the arguments that define the test program. The +/// special argument 'test_suite' provides an override to the global test suite +/// name. The rest of the arguments are part of the test program metadata. +/// \pre state(upvalue 1) String with the name of the interface. +/// \pre state(upvalue 2) User configuration with the per-test suite settings. +/// \pre state(upvalue 3) Scheduler context to run test programs in. +/// +/// \param state The Lua state that executed the function. +/// +/// \return Number of return values left on the Lua stack. +/// +/// \throw std::runtime_error If the arguments to the function are invalid. +static int +lua_generic_test_program(lutok::state& state) +{ + if (!state.is_string(state.upvalue_index(1))) + throw std::runtime_error("Found corrupt state for test_program " + "function"); + const std::string interface = state.to_string(state.upvalue_index(1)); + + if (!state.is_userdata(state.upvalue_index(2))) + throw std::runtime_error("Found corrupt state for test_program " + "function"); + const config::tree* user_config = *state.to_userdata< const config::tree* >( + state.upvalue_index(2)); + + if (!state.is_userdata(state.upvalue_index(3))) + throw std::runtime_error("Found corrupt state for test_program " + "function"); + scheduler::scheduler_handle* scheduler_handle = + *state.to_userdata< scheduler::scheduler_handle* >( + state.upvalue_index(3)); + + if (!state.is_table(-1)) + throw std::runtime_error( + F("%s_test_program expects a table of properties as its single " + "argument") % interface); + + scheduler::ensure_valid_interface(interface); + + lutok::stack_cleaner cleaner(state); + + state.push_string("name"); + state.get_table(-2); + if (!state.is_string(-1)) + throw std::runtime_error("Test program name not defined or not a " + "string"); + const fs::path path(state.to_string(-1)); + state.pop(1); + + state.push_string("test_suite"); + state.get_table(-2); + std::string test_suite; + if (state.is_nil(-1)) { + // Leave empty to use the global test-suite value. + } else if (state.is_string(-1)) { + test_suite = state.to_string(-1); + } else { + throw std::runtime_error(F("Found non-string value in the test_suite " + "property of test program '%s'") % path); + } + state.pop(1); + + model::metadata_builder mdbuilder; + state.push_nil(); + while (state.next(-2)) { + if (!state.is_string(-2)) + throw std::runtime_error(F("Found non-string metadata property " + "name in test program '%s'") % + path); + const std::string property = state.to_string(-2); + + if (property != "name" && property != "test_suite") { + std::string value; + if (state.is_boolean(-1)) { + value = F("%s") % state.to_boolean(-1); + } else if (state.is_number(-1)) { + value = F("%s") % state.to_integer(-1); + } else if (state.is_string(-1)) { + value = state.to_string(-1); + } else { + throw std::runtime_error( + F("Metadata property '%s' in test program '%s' cannot be " + "converted to a string") % property % path); + } + + mdbuilder.set_string(property, value); + } + + state.pop(1); + } + + parser::get_from_state(state)->callback_test_program( + interface, path, test_suite, mdbuilder.build(), *user_config, + *scheduler_handle); + return 0; +} + + +/// Glue to invoke parser::callback_current_kyuafile() from Lua. +/// +/// \param state The Lua state that executed the function. +/// +/// \return Number of return values left on the Lua stack. +static int +lua_current_kyuafile(lutok::state& state) +{ + state.push_string(parser::get_from_state(state)-> + callback_current_kyuafile().str()); + return 1; +} + + +/// Glue to invoke parser::callback_include() from Lua. +/// +/// \param state The Lua state that executed the function. +/// +/// \pre state(upvalue 1) User configuration with the per-test suite settings. +/// \pre state(upvalue 2) Scheduler context to run test programs in. +/// +/// \return Number of return values left on the Lua stack. +static int +lua_include(lutok::state& state) +{ + if (!state.is_userdata(state.upvalue_index(1))) + throw std::runtime_error("Found corrupt state for test_program " + "function"); + const config::tree* user_config = *state.to_userdata< const config::tree* >( + state.upvalue_index(1)); + + if (!state.is_userdata(state.upvalue_index(2))) + throw std::runtime_error("Found corrupt state for test_program " + "function"); + scheduler::scheduler_handle* scheduler_handle = + *state.to_userdata< scheduler::scheduler_handle* >( + state.upvalue_index(2)); + + parser::get_from_state(state)->callback_include( + fs::path(state.to_string(-1)), *user_config, *scheduler_handle); + return 0; +} + + +/// Glue to invoke parser::callback_syntax() from Lua. +/// +/// \pre state(-2) The syntax format name, if a v1 file. +/// \pre state(-1) The syntax format version. +/// +/// \param state The Lua state that executed the function. +/// +/// \return Number of return values left on the Lua stack. +static int +lua_syntax(lutok::state& state) +{ + if (!state.is_number(-1)) + throw std::runtime_error("Last argument to syntax must be a number"); + const int syntax_version = state.to_integer(-1); + + if (syntax_version == 1) { + if (state.get_top() != 2) + throw std::runtime_error("Version 1 files need two arguments to " + "syntax()"); + if (!state.is_string(-2) || state.to_string(-2) != "kyuafile") + throw std::runtime_error("First argument to syntax must be " + "'kyuafile' for version 1 files"); + } else { + if (state.get_top() != 1) + throw std::runtime_error("syntax() only takes one argument"); + } + + parser::get_from_state(state)->callback_syntax(syntax_version); + return 0; +} + + +/// Glue to invoke parser::callback_test_suite() from Lua. +/// +/// \param state The Lua state that executed the function. +/// +/// \return Number of return values left on the Lua stack. +static int +lua_test_suite(lutok::state& state) +{ + parser::get_from_state(state)->callback_test_suite(state.to_string(-1)); + return 0; +} + + +} // anonymous namespace + + +/// Constructs a kyuafile form initialized data. +/// +/// Use load() to parse a test suite configuration file and construct a +/// kyuafile object. +/// +/// \param source_root_ The root directory for the test suite represented by the +/// Kyuafile. In other words, the directory containing the first Kyuafile +/// processed. +/// \param build_root_ The root directory for the test programs themselves. In +/// general, this will be the same as source_root_. If different, the +/// specified directory must follow the exact same layout of source_root_. +/// \param tps_ Collection of test programs that belong to this test suite. +engine::kyuafile::kyuafile(const fs::path& source_root_, + const fs::path& build_root_, + const model::test_programs_vector& tps_) : + _source_root(source_root_), + _build_root(build_root_), + _test_programs(tps_) +{ +} + + +/// Destructor. +engine::kyuafile::~kyuafile(void) +{ +} + + +/// Parses a test suite configuration file. +/// +/// \param file The file to parse. +/// \param user_build_root If not none, specifies a path to a directory +/// containing the test programs themselves. The layout of the build root +/// must match the layout of the source root (which is just the directory +/// from which the Kyuafile is being read). +/// \param user_config User configuration holding any test suite properties +/// to be passed to the list operation. +/// \param scheduler_handle The scheduler context to use for loading the test +/// case lists. +/// +/// \return High-level representation of the configuration file. +/// +/// \throw load_error If there is any problem loading the file. This includes +/// file access errors and syntax errors. +engine::kyuafile +engine::kyuafile::load(const fs::path& file, + const optional< fs::path > user_build_root, + const config::tree& user_config, + scheduler::scheduler_handle& scheduler_handle) +{ + const fs::path source_root_ = file.branch_path(); + const fs::path build_root_ = user_build_root ? + user_build_root.get() : source_root_; + + // test_program.absolute_path() uses the current work directory and that + // fails to resolve the correct path once we have used chdir to enter the + // test work directory. To prevent this causing issues down the road, + // force the build root to be absolute so that absolute_path() does not + // need to rely on the current work directory. + const fs::path abs_build_root = build_root_.is_absolute() ? + build_root_ : build_root_.to_absolute(); + + return kyuafile(source_root_, build_root_, + parser(source_root_, abs_build_root, + fs::path(file.leaf_name()), user_config, + scheduler_handle).parse()); +} + + +/// Gets the root directory of the test suite. +/// +/// \return A path. +const fs::path& +engine::kyuafile::source_root(void) const +{ + return _source_root; +} + + +/// Gets the root directory of the test programs. +/// +/// \return A path. +const fs::path& +engine::kyuafile::build_root(void) const +{ + return _build_root; +} + + +/// Gets the collection of test programs that belong to this test suite. +/// +/// \return Collection of test program executable names. +const model::test_programs_vector& +engine::kyuafile::test_programs(void) const +{ + return _test_programs; +} diff --git a/engine/kyuafile.hpp b/engine/kyuafile.hpp new file mode 100644 index 000000000000..161f4305f4d1 --- /dev/null +++ b/engine/kyuafile.hpp @@ -0,0 +1,96 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/kyuafile.hpp +/// Test suite configuration parsing and representation. + +#if !defined(ENGINE_KYUAFILE_HPP) +#define ENGINE_KYUAFILE_HPP + +#include "engine/kyuafile_fwd.hpp" + +#include <string> +#include <vector> + +#include <lutok/state.hpp> + +#include "engine/scheduler_fwd.hpp" +#include "model/test_program_fwd.hpp" +#include "utils/config/tree_fwd.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional_fwd.hpp" + +namespace engine { + + +/// Representation of the configuration of a test suite. +/// +/// Test suites are collections of related test programs. They are described by +/// a configuration file. +/// +/// Test suites have two path references: one to the "source root" and another +/// one to the "build root". The source root points to the directory from which +/// the Kyuafile is being read, and all recursive inclusions are resolved +/// relative to that directory. The build root points to the directory +/// containing the generated test programs and is prepended to the absolute path +/// of the test programs referenced by the Kyuafiles. In general, the build +/// root will be the same as the source root; however, when using a build system +/// that supports "build directories", providing this option comes in handy to +/// allow running the tests without much hassle. +/// +/// This class provides the parser for test suite configuration files and +/// methods to access the parsed data. +class kyuafile { + /// Path to the directory containing the top-level Kyuafile loaded. + utils::fs::path _source_root; + + /// Path to the directory containing the test programs. + utils::fs::path _build_root; + + /// Collection of the test programs defined in the Kyuafile. + model::test_programs_vector _test_programs; + +public: + explicit kyuafile(const utils::fs::path&, const utils::fs::path&, + const model::test_programs_vector&); + ~kyuafile(void); + + static kyuafile load(const utils::fs::path&, + const utils::optional< utils::fs::path >, + const utils::config::tree&, + scheduler::scheduler_handle&); + + const utils::fs::path& source_root(void) const; + const utils::fs::path& build_root(void) const; + const model::test_programs_vector& test_programs(void) const; +}; + + +} // namespace engine + +#endif // !defined(ENGINE_KYUAFILE_HPP) diff --git a/engine/kyuafile_fwd.hpp b/engine/kyuafile_fwd.hpp new file mode 100644 index 000000000000..60a98f65e3ab --- /dev/null +++ b/engine/kyuafile_fwd.hpp @@ -0,0 +1,43 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/kyuafile_fwd.hpp +/// Forward declarations for engine/kyuafile.hpp + +#if !defined(ENGINE_KYUAFILE_FWD_HPP) +#define ENGINE_KYUAFILE_FWD_HPP + +namespace engine { + + +class kyuafile; + + +} // namespace engine + +#endif // !defined(ENGINE_KYUAFILE_FWD_HPP) diff --git a/engine/kyuafile_test.cpp b/engine/kyuafile_test.cpp new file mode 100644 index 000000000000..d95f28c71acb --- /dev/null +++ b/engine/kyuafile_test.cpp @@ -0,0 +1,606 @@ +// Copyright 2010 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/kyuafile.hpp" + +extern "C" { +#include <unistd.h> +} + +#include <stdexcept> +#include <typeinfo> + +#include <atf-c++.hpp> +#include <lutok/operations.hpp> +#include <lutok/state.ipp> +#include <lutok/test_utils.hpp> + +#include "engine/atf.hpp" +#include "engine/exceptions.hpp" +#include "engine/plain.hpp" +#include "engine/scheduler.hpp" +#include "engine/tap.hpp" +#include "model/metadata.hpp" +#include "model/test_program.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/env.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/optional.ipp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace scheduler = engine::scheduler; + +using utils::none; + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__empty); +ATF_TEST_CASE_BODY(kyuafile__load__empty) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + atf::utils::create_file("config", "syntax(2)\n"); + + const engine::kyuafile suite = engine::kyuafile::load( + fs::path("config"), none, config::tree(), handle); + ATF_REQUIRE_EQ(fs::path("."), suite.source_root()); + ATF_REQUIRE_EQ(fs::path("."), suite.build_root()); + ATF_REQUIRE_EQ(0, suite.test_programs().size()); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__real_interfaces); +ATF_TEST_CASE_BODY(kyuafile__load__real_interfaces) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + atf::utils::create_file( + "config", + "syntax(2)\n" + "test_suite('one-suite')\n" + "atf_test_program{name='1st'}\n" + "atf_test_program{name='2nd', test_suite='first'}\n" + "plain_test_program{name='3rd'}\n" + "tap_test_program{name='4th', test_suite='second'}\n" + "include('dir/config')\n"); + + fs::mkdir(fs::path("dir"), 0755); + atf::utils::create_file( + "dir/config", + "syntax(2)\n" + "atf_test_program{name='1st', test_suite='other-suite'}\n" + "include('subdir/config')\n"); + + fs::mkdir(fs::path("dir/subdir"), 0755); + atf::utils::create_file( + "dir/subdir/config", + "syntax(2)\n" + "atf_test_program{name='5th', test_suite='last-suite'}\n"); + + atf::utils::create_file("1st", ""); + atf::utils::create_file("2nd", ""); + atf::utils::create_file("3rd", ""); + atf::utils::create_file("4th", ""); + atf::utils::create_file("dir/1st", ""); + atf::utils::create_file("dir/subdir/5th", ""); + + const engine::kyuafile suite = engine::kyuafile::load( + fs::path("config"), none, config::tree(), handle); + ATF_REQUIRE_EQ(fs::path("."), suite.source_root()); + ATF_REQUIRE_EQ(fs::path("."), suite.build_root()); + ATF_REQUIRE_EQ(6, suite.test_programs().size()); + + ATF_REQUIRE_EQ("atf", suite.test_programs()[0]->interface_name()); + ATF_REQUIRE_EQ(fs::path("1st"), suite.test_programs()[0]->relative_path()); + ATF_REQUIRE_EQ("one-suite", suite.test_programs()[0]->test_suite_name()); + + ATF_REQUIRE_EQ("atf", suite.test_programs()[1]->interface_name()); + ATF_REQUIRE_EQ(fs::path("2nd"), suite.test_programs()[1]->relative_path()); + ATF_REQUIRE_EQ("first", suite.test_programs()[1]->test_suite_name()); + + ATF_REQUIRE_EQ("plain", suite.test_programs()[2]->interface_name()); + ATF_REQUIRE_EQ(fs::path("3rd"), suite.test_programs()[2]->relative_path()); + ATF_REQUIRE_EQ("one-suite", suite.test_programs()[2]->test_suite_name()); + + ATF_REQUIRE_EQ("tap", suite.test_programs()[3]->interface_name()); + ATF_REQUIRE_EQ(fs::path("4th"), suite.test_programs()[3]->relative_path()); + ATF_REQUIRE_EQ("second", suite.test_programs()[3]->test_suite_name()); + + ATF_REQUIRE_EQ("atf", suite.test_programs()[4]->interface_name()); + ATF_REQUIRE_EQ(fs::path("dir/1st"), + suite.test_programs()[4]->relative_path()); + ATF_REQUIRE_EQ("other-suite", suite.test_programs()[4]->test_suite_name()); + + ATF_REQUIRE_EQ("atf", suite.test_programs()[5]->interface_name()); + ATF_REQUIRE_EQ(fs::path("dir/subdir/5th"), + suite.test_programs()[5]->relative_path()); + ATF_REQUIRE_EQ("last-suite", suite.test_programs()[5]->test_suite_name()); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__mock_interfaces); +ATF_TEST_CASE_BODY(kyuafile__load__mock_interfaces) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + std::shared_ptr< scheduler::interface > mock_interface( + new engine::plain_interface()); + + scheduler::register_interface("some", mock_interface); + scheduler::register_interface("random", mock_interface); + scheduler::register_interface("names", mock_interface); + + atf::utils::create_file( + "config", + "syntax(2)\n" + "test_suite('one-suite')\n" + "some_test_program{name='1st'}\n" + "random_test_program{name='2nd'}\n" + "names_test_program{name='3rd'}\n"); + + atf::utils::create_file("1st", ""); + atf::utils::create_file("2nd", ""); + atf::utils::create_file("3rd", ""); + + const engine::kyuafile suite = engine::kyuafile::load( + fs::path("config"), none, config::tree(), handle); + ATF_REQUIRE_EQ(fs::path("."), suite.source_root()); + ATF_REQUIRE_EQ(fs::path("."), suite.build_root()); + ATF_REQUIRE_EQ(3, suite.test_programs().size()); + + ATF_REQUIRE_EQ("some", suite.test_programs()[0]->interface_name()); + ATF_REQUIRE_EQ(fs::path("1st"), suite.test_programs()[0]->relative_path()); + + ATF_REQUIRE_EQ("random", suite.test_programs()[1]->interface_name()); + ATF_REQUIRE_EQ(fs::path("2nd"), suite.test_programs()[1]->relative_path()); + + ATF_REQUIRE_EQ("names", suite.test_programs()[2]->interface_name()); + ATF_REQUIRE_EQ(fs::path("3rd"), suite.test_programs()[2]->relative_path()); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__metadata); +ATF_TEST_CASE_BODY(kyuafile__load__metadata) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + atf::utils::create_file( + "config", + "syntax(2)\n" + "atf_test_program{name='1st', test_suite='first'," + " allowed_architectures='amd64 i386', timeout=15}\n" + "plain_test_program{name='2nd', test_suite='second'," + " required_files='foo /bar//baz', required_user='root'," + " ['custom.a-number']=123, ['custom.a-bool']=true}\n"); + atf::utils::create_file("1st", ""); + atf::utils::create_file("2nd", ""); + + const engine::kyuafile suite = engine::kyuafile::load( + fs::path("config"), none, config::tree(), handle); + ATF_REQUIRE_EQ(2, suite.test_programs().size()); + + ATF_REQUIRE_EQ("atf", suite.test_programs()[0]->interface_name()); + ATF_REQUIRE_EQ(fs::path("1st"), suite.test_programs()[0]->relative_path()); + ATF_REQUIRE_EQ("first", suite.test_programs()[0]->test_suite_name()); + const model::metadata md1 = model::metadata_builder() + .add_allowed_architecture("amd64") + .add_allowed_architecture("i386") + .set_timeout(datetime::delta(15, 0)) + .build(); + ATF_REQUIRE_EQ(md1, suite.test_programs()[0]->get_metadata()); + + ATF_REQUIRE_EQ("plain", suite.test_programs()[1]->interface_name()); + ATF_REQUIRE_EQ(fs::path("2nd"), suite.test_programs()[1]->relative_path()); + ATF_REQUIRE_EQ("second", suite.test_programs()[1]->test_suite_name()); + const model::metadata md2 = model::metadata_builder() + .add_required_file(fs::path("foo")) + .add_required_file(fs::path("/bar/baz")) + .add_custom("a-bool", "true") + .add_custom("a-number", "123") + .set_required_user("root") + .build(); + ATF_REQUIRE_EQ(md2, suite.test_programs()[1]->get_metadata()); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__current_directory); +ATF_TEST_CASE_BODY(kyuafile__load__current_directory) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + atf::utils::create_file( + "config", + "syntax(2)\n" + "atf_test_program{name='one', test_suite='first'}\n" + "include('config2')\n"); + + atf::utils::create_file( + "config2", + "syntax(2)\n" + "test_suite('second')\n" + "atf_test_program{name='two'}\n"); + + atf::utils::create_file("one", ""); + atf::utils::create_file("two", ""); + + const engine::kyuafile suite = engine::kyuafile::load( + fs::path("config"), none, config::tree(), handle); + ATF_REQUIRE_EQ(fs::path("."), suite.source_root()); + ATF_REQUIRE_EQ(fs::path("."), suite.build_root()); + ATF_REQUIRE_EQ(2, suite.test_programs().size()); + ATF_REQUIRE_EQ(fs::path("one"), suite.test_programs()[0]->relative_path()); + ATF_REQUIRE_EQ("first", suite.test_programs()[0]->test_suite_name()); + ATF_REQUIRE_EQ(fs::path("two"), + suite.test_programs()[1]->relative_path()); + ATF_REQUIRE_EQ("second", suite.test_programs()[1]->test_suite_name()); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__other_directory); +ATF_TEST_CASE_BODY(kyuafile__load__other_directory) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + fs::mkdir(fs::path("root"), 0755); + atf::utils::create_file( + "root/config", + "syntax(2)\n" + "test_suite('abc')\n" + "atf_test_program{name='one'}\n" + "include('dir/config')\n"); + + fs::mkdir(fs::path("root/dir"), 0755); + atf::utils::create_file( + "root/dir/config", + "syntax(2)\n" + "test_suite('foo')\n" + "atf_test_program{name='two', test_suite='def'}\n" + "atf_test_program{name='three'}\n"); + + atf::utils::create_file("root/one", ""); + atf::utils::create_file("root/dir/two", ""); + atf::utils::create_file("root/dir/three", ""); + + const engine::kyuafile suite = engine::kyuafile::load( + fs::path("root/config"), none, config::tree(), handle); + ATF_REQUIRE_EQ(fs::path("root"), suite.source_root()); + ATF_REQUIRE_EQ(fs::path("root"), suite.build_root()); + ATF_REQUIRE_EQ(3, suite.test_programs().size()); + ATF_REQUIRE_EQ(fs::path("one"), suite.test_programs()[0]->relative_path()); + ATF_REQUIRE_EQ("abc", suite.test_programs()[0]->test_suite_name()); + ATF_REQUIRE_EQ(fs::path("dir/two"), + suite.test_programs()[1]->relative_path()); + ATF_REQUIRE_EQ("def", suite.test_programs()[1]->test_suite_name()); + ATF_REQUIRE_EQ(fs::path("dir/three"), + suite.test_programs()[2]->relative_path()); + ATF_REQUIRE_EQ("foo", suite.test_programs()[2]->test_suite_name()); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__build_directory); +ATF_TEST_CASE_BODY(kyuafile__load__build_directory) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + fs::mkdir(fs::path("srcdir"), 0755); + atf::utils::create_file( + "srcdir/config", + "syntax(2)\n" + "test_suite('abc')\n" + "atf_test_program{name='one'}\n" + "include('dir/config')\n"); + + fs::mkdir(fs::path("srcdir/dir"), 0755); + atf::utils::create_file( + "srcdir/dir/config", + "syntax(2)\n" + "test_suite('foo')\n" + "atf_test_program{name='two', test_suite='def'}\n" + "atf_test_program{name='three'}\n"); + + fs::mkdir(fs::path("builddir"), 0755); + atf::utils::create_file("builddir/one", ""); + fs::mkdir(fs::path("builddir/dir"), 0755); + atf::utils::create_file("builddir/dir/two", ""); + atf::utils::create_file("builddir/dir/three", ""); + + const engine::kyuafile suite = engine::kyuafile::load( + fs::path("srcdir/config"), utils::make_optional(fs::path("builddir")), + config::tree(), handle); + ATF_REQUIRE_EQ(fs::path("srcdir"), suite.source_root()); + ATF_REQUIRE_EQ(fs::path("builddir"), suite.build_root()); + ATF_REQUIRE_EQ(3, suite.test_programs().size()); + ATF_REQUIRE_EQ(fs::path("builddir/one").to_absolute(), + suite.test_programs()[0]->absolute_path()); + ATF_REQUIRE_EQ(fs::path("one"), suite.test_programs()[0]->relative_path()); + ATF_REQUIRE_EQ("abc", suite.test_programs()[0]->test_suite_name()); + ATF_REQUIRE_EQ(fs::path("builddir/dir/two").to_absolute(), + suite.test_programs()[1]->absolute_path()); + ATF_REQUIRE_EQ(fs::path("dir/two"), + suite.test_programs()[1]->relative_path()); + ATF_REQUIRE_EQ("def", suite.test_programs()[1]->test_suite_name()); + ATF_REQUIRE_EQ(fs::path("builddir/dir/three").to_absolute(), + suite.test_programs()[2]->absolute_path()); + ATF_REQUIRE_EQ(fs::path("dir/three"), + suite.test_programs()[2]->relative_path()); + ATF_REQUIRE_EQ("foo", suite.test_programs()[2]->test_suite_name()); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__absolute_paths_are_stable); +ATF_TEST_CASE_BODY(kyuafile__load__absolute_paths_are_stable) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + atf::utils::create_file( + "config", + "syntax(2)\n" + "atf_test_program{name='one', test_suite='first'}\n"); + atf::utils::create_file("one", ""); + + const engine::kyuafile suite = engine::kyuafile::load( + fs::path("config"), none, config::tree(), handle); + + const fs::path previous_dir = fs::current_path(); + fs::mkdir(fs::path("other"), 0755); + // Change the directory. We want later calls to absolute_path() on the test + // programs to return references to previous_dir instead. + ATF_REQUIRE(::chdir("other") != -1); + + ATF_REQUIRE_EQ(fs::path("."), suite.source_root()); + ATF_REQUIRE_EQ(fs::path("."), suite.build_root()); + ATF_REQUIRE_EQ(1, suite.test_programs().size()); + ATF_REQUIRE_EQ(previous_dir / "one", + suite.test_programs()[0]->absolute_path()); + ATF_REQUIRE_EQ(fs::path("one"), suite.test_programs()[0]->relative_path()); + ATF_REQUIRE_EQ("first", suite.test_programs()[0]->test_suite_name()); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__fs_calls_are_relative); +ATF_TEST_CASE_BODY(kyuafile__load__fs_calls_are_relative) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + atf::utils::create_file( + "Kyuafile", + "syntax(2)\n" + "if fs.exists('one') then\n" + " plain_test_program{name='one', test_suite='first'}\n" + "end\n" + "if fs.exists('two') then\n" + " plain_test_program{name='two', test_suite='first'}\n" + "end\n" + "include('dir/Kyuafile')\n"); + atf::utils::create_file("one", ""); + fs::mkdir(fs::path("dir"), 0755); + atf::utils::create_file( + "dir/Kyuafile", + "syntax(2)\n" + "if fs.exists('one') then\n" + " plain_test_program{name='one', test_suite='first'}\n" + "end\n" + "if fs.exists('two') then\n" + " plain_test_program{name='two', test_suite='first'}\n" + "end\n"); + atf::utils::create_file("dir/two", ""); + + const engine::kyuafile suite = engine::kyuafile::load( + fs::path("Kyuafile"), none, config::tree(), handle); + + ATF_REQUIRE_EQ(2, suite.test_programs().size()); + ATF_REQUIRE_EQ(fs::current_path() / "one", + suite.test_programs()[0]->absolute_path()); + ATF_REQUIRE_EQ(fs::current_path() / "dir/two", + suite.test_programs()[1]->absolute_path()); + + handle.cleanup(); +} + + +/// Verifies that load raises a load_error on a given input. +/// +/// \param file Name of the file to load. +/// \param regex Expression to match on load_error's contents. +static void +do_load_error_test(const char* file, const char* regex) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + ATF_REQUIRE_THROW_RE(engine::load_error, regex, + engine::kyuafile::load(fs::path(file), none, + config::tree(), handle)); + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__test_program_not_basename); +ATF_TEST_CASE_BODY(kyuafile__load__test_program_not_basename) +{ + atf::utils::create_file( + "config", + "syntax(2)\n" + "test_suite('abc')\n" + "atf_test_program{name='one'}\n" + "atf_test_program{name='./ls'}\n"); + + atf::utils::create_file("one", ""); + do_load_error_test("config", "./ls.*path components"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__lua_error); +ATF_TEST_CASE_BODY(kyuafile__load__lua_error) +{ + atf::utils::create_file("config", "this syntax is invalid\n"); + + do_load_error_test("config", ".*"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__syntax__not_called); +ATF_TEST_CASE_BODY(kyuafile__load__syntax__not_called) +{ + atf::utils::create_file("config", ""); + + do_load_error_test("config", "syntax.* never called"); +} + + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__syntax__deprecated_format); +ATF_TEST_CASE_BODY(kyuafile__load__syntax__deprecated_format) +{ + atf::utils::create_file("config", "syntax('foo', 1)\n"); + do_load_error_test("config", "must be 'kyuafile'"); + + atf::utils::create_file("config", "syntax('config', 2)\n"); + do_load_error_test("config", "only takes one argument"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__syntax__twice); +ATF_TEST_CASE_BODY(kyuafile__load__syntax__twice) +{ + atf::utils::create_file( + "config", + "syntax(2)\n" + "syntax(2)\n"); + + do_load_error_test("config", "Can only call syntax.* once"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__syntax__bad_version); +ATF_TEST_CASE_BODY(kyuafile__load__syntax__bad_version) +{ + atf::utils::create_file("config", "syntax(12)\n"); + + do_load_error_test("config", "Unsupported file version 12"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__test_suite__missing); +ATF_TEST_CASE_BODY(kyuafile__load__test_suite__missing) +{ + atf::utils::create_file( + "config", + "syntax(2)\n" + "plain_test_program{name='one'}"); + + atf::utils::create_file("one", ""); + + do_load_error_test("config", "No test suite defined"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__test_suite__twice); +ATF_TEST_CASE_BODY(kyuafile__load__test_suite__twice) +{ + atf::utils::create_file( + "config", + "syntax(2)\n" + "test_suite('foo')\n" + "test_suite('bar')\n"); + + do_load_error_test("config", "Can only call test_suite.* once"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__missing_file); +ATF_TEST_CASE_BODY(kyuafile__load__missing_file) +{ + do_load_error_test("missing", "Load of 'missing' failed"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(kyuafile__load__missing_test_program); +ATF_TEST_CASE_BODY(kyuafile__load__missing_test_program) +{ + atf::utils::create_file( + "config", + "syntax(2)\n" + "atf_test_program{name='one', test_suite='first'}\n" + "atf_test_program{name='two', test_suite='first'}\n"); + + atf::utils::create_file("one", ""); + + do_load_error_test("config", "Non-existent.*'two'"); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + scheduler::register_interface( + "atf", std::shared_ptr< scheduler::interface >( + new engine::atf_interface())); + scheduler::register_interface( + "plain", std::shared_ptr< scheduler::interface >( + new engine::plain_interface())); + scheduler::register_interface( + "tap", std::shared_ptr< scheduler::interface >( + new engine::tap_interface())); + + ATF_ADD_TEST_CASE(tcs, kyuafile__load__empty); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__real_interfaces); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__mock_interfaces); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__metadata); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__current_directory); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__other_directory); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__build_directory); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__absolute_paths_are_stable); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__fs_calls_are_relative); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__test_program_not_basename); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__lua_error); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__syntax__not_called); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__syntax__deprecated_format); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__syntax__twice); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__syntax__bad_version); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__test_suite__missing); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__test_suite__twice); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__missing_file); + ATF_ADD_TEST_CASE(tcs, kyuafile__load__missing_test_program); +} diff --git a/engine/plain.cpp b/engine/plain.cpp new file mode 100644 index 000000000000..8346e50bbecf --- /dev/null +++ b/engine/plain.cpp @@ -0,0 +1,143 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/plain.hpp" + +extern "C" { +#include <unistd.h> +} + +#include <cstdlib> + +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/defs.hpp" +#include "utils/env.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/process/operations.hpp" +#include "utils/process/status.hpp" +#include "utils/sanity.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace process = utils::process; + +using utils::optional; + + +/// Executes a test program's list operation. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +void +engine::plain_interface::exec_list( + const model::test_program& /* test_program */, + const config::properties_map& /* vars */) const +{ + ::_exit(EXIT_SUCCESS); +} + + +/// Computes the test cases list of a test program. +/// +/// \return A list of test cases. +model::test_cases_map +engine::plain_interface::parse_list( + const optional< process::status >& /* status */, + const fs::path& /* stdout_path */, + const fs::path& /* stderr_path */) const +{ + return model::test_cases_map_builder().add("main").build(); +} + + +/// Executes a test case of the test program. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +/// +/// \param test_program The test program to execute. +/// \param test_case_name Name of the test case to invoke. +/// \param vars User-provided variables to pass to the test program. +void +engine::plain_interface::exec_test( + const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars, + const fs::path& /* control_directory */) const +{ + PRE(test_case_name == "main"); + + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + utils::setenv(F("TEST_ENV_%s") % (*iter).first, (*iter).second); + } + + process::args_vector args; + process::exec(test_program.absolute_path(), args); +} + + +/// Computes the result of a test case based on its termination status. +/// +/// \param status The termination status of the subprocess used to execute +/// the exec_test() method or none if the test timed out. +/// +/// \return A test result. +model::test_result +engine::plain_interface::compute_result( + const optional< process::status >& status, + const fs::path& /* control_directory */, + const fs::path& /* stdout_path */, + const fs::path& /* stderr_path */) const +{ + if (!status) { + return model::test_result(model::test_result_broken, + "Test case timed out"); + } + + if (status.get().exited()) { + const int exitstatus = status.get().exitstatus(); + if (exitstatus == EXIT_SUCCESS) { + return model::test_result(model::test_result_passed); + } else { + return model::test_result( + model::test_result_failed, + F("Returned non-success exit status %s") % exitstatus); + } + } else { + return model::test_result( + model::test_result_broken, + F("Received signal %s") % status.get().termsig()); + } +} diff --git a/engine/plain.hpp b/engine/plain.hpp new file mode 100644 index 000000000000..ee5f3e746781 --- /dev/null +++ b/engine/plain.hpp @@ -0,0 +1,67 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/plain.hpp +/// Execution engine for test programs that implement the plain interface. + +#if !defined(ENGINE_PLAIN_HPP) +#define ENGINE_PLAIN_HPP + +#include "engine/scheduler.hpp" + +namespace engine { + + +/// Implementation of the scheduler interface for plain test programs. +class plain_interface : public engine::scheduler::interface { +public: + void exec_list(const model::test_program&, + const utils::config::properties_map&) const UTILS_NORETURN; + + model::test_cases_map parse_list( + const utils::optional< utils::process::status >&, + const utils::fs::path&, + const utils::fs::path&) const; + + void exec_test(const model::test_program&, const std::string&, + const utils::config::properties_map&, + const utils::fs::path&) const + UTILS_NORETURN; + + model::test_result compute_result( + const utils::optional< utils::process::status >&, + const utils::fs::path&, + const utils::fs::path&, + const utils::fs::path&) const; +}; + + +} // namespace engine + + +#endif // !defined(ENGINE_PLAIN_HPP) diff --git a/engine/plain_helpers.cpp b/engine/plain_helpers.cpp new file mode 100644 index 000000000000..52b1bc74fe10 --- /dev/null +++ b/engine/plain_helpers.cpp @@ -0,0 +1,238 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include <sys/stat.h> + +#include <unistd.h> + +extern char** environ; +} + +#include <cstdlib> +#include <cstring> +#include <fstream> +#include <iostream> +#include <sstream> + +#include "utils/env.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/test_utils.ipp" + +namespace fs = utils::fs; + +using utils::optional; + + +namespace { + + +/// Gets the name of the test case to run. +/// +/// We use the value of the TEST_CASE environment variable if present, or +/// else the basename of the test program. +/// +/// \param arg0 Value of argv[0] as passed to main(). +/// +/// \return A test case name. The name may not be valid. +static std::string +guess_test_case_name(const char* arg0) +{ + const optional< std::string > test_case_env = utils::getenv("TEST_CASE"); + if (test_case_env) { + return test_case_env.get(); + } else { + return fs::path(arg0).leaf_name(); + } +} + + +/// Logs an error message and exits the test with an error code. +/// +/// \param str The error message to log. +static void +fail(const std::string& str) +{ + std::cerr << str << '\n'; + std::exit(EXIT_FAILURE); +} + + +/// A test case that validates the TEST_ENV_* variables. +static void +test_check_configuration_variables(void) +{ + std::set< std::string > vars; + char** iter; + for (iter = environ; *iter != NULL; ++iter) { + if (std::strstr(*iter, "TEST_ENV_") == *iter) { + vars.insert(*iter); + } + } + + std::set< std::string > exp_vars; + exp_vars.insert("TEST_ENV_first=some value"); + exp_vars.insert("TEST_ENV_second=some other value"); + if (vars != exp_vars) { + fail(F("Expected: %s\nFound: %s\n") % exp_vars % vars); + } +} + + +/// A test case that crashes. +static void +test_crash(void) +{ + utils::abort_without_coredump(); +} + + +/// A test case that exits with a non-zero exit code, and not 1. +static void +test_fail(void) +{ + std::exit(8); +} + + +/// A test case that passes. +static void +test_pass(void) +{ +} + + +/// A test case that spawns a subchild that gets stuck. +/// +/// This test case is used by the caller to validate that the whole process tree +/// is terminated when the test case is killed. +static void +test_spawn_blocking_child(void) +{ + pid_t pid = ::fork(); + if (pid == -1) + fail("Cannot fork subprocess"); + else if (pid == 0) { + for (;;) + ::pause(); + } else { + const fs::path name = fs::path(utils::getenv("CONTROL_DIR").get()) / + "pid"; + std::ofstream pidfile(name.c_str()); + if (!pidfile) + fail("Failed to create the pidfile"); + pidfile << pid; + pidfile.close(); + } +} + + +/// A test case that times out. +/// +/// Note that the timeout is defined in the Kyuafile, as the plain interface has +/// no means for test programs to specify this by themselves. +static void +test_timeout(void) +{ + ::sleep(10); + const fs::path control_dir = fs::path(utils::getenv("CONTROL_DIR").get()); + std::ofstream file((control_dir / "cookie").c_str()); + if (!file) + fail("Failed to create the control cookie"); + file.close(); +} + + +/// A test case that performs basic checks on the runtime environment. +/// +/// If the runtime environment does not look clean (according to the rules in +/// the Kyua runtime properties), the test fails. +static void +test_validate_isolation(void) +{ + if (utils::getenv("HOME").get() == "fake-value") + fail("HOME not reset"); + if (utils::getenv("LANG")) + fail("LANG not unset"); +} + + +} // anonymous namespace + + +/// Entry point to the test program. +/// +/// The caller can select which test case to run by defining the TEST_CASE +/// environment variable. This is not "standard", in the sense this is not a +/// generic property of the plain test case interface. +/// +/// \todo It may be worth to split this binary into separate, smaller binaries, +/// one for every "test case". We use this program as a dispatcher for +/// different "main"s, the only reason being to keep the amount of helper test +/// programs to a minimum. However, putting this each function in its own +/// binary could simplify many other things. +/// +/// \param argc The number of CLI arguments. +/// \param argv The CLI arguments themselves. These are not used because +/// Kyua will not pass any arguments to the plain test program. +int +main(int argc, char** argv) +{ + if (argc != 1) { + std::cerr << "No arguments allowed; select the test case with the " + "TEST_CASE variable"; + return EXIT_FAILURE; + } + + const std::string& test_case = guess_test_case_name(argv[0]); + + if (test_case == "check_configuration_variables") + test_check_configuration_variables(); + else if (test_case == "crash") + test_crash(); + else if (test_case == "fail") + test_fail(); + else if (test_case == "pass") + test_pass(); + else if (test_case == "spawn_blocking_child") + test_spawn_blocking_child(); + else if (test_case == "timeout") + test_timeout(); + else if (test_case == "validate_isolation") + test_validate_isolation(); + else { + std::cerr << "Unknown test case"; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/engine/plain_test.cpp b/engine/plain_test.cpp new file mode 100644 index 000000000000..cc3326e4c581 --- /dev/null +++ b/engine/plain_test.cpp @@ -0,0 +1,207 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/plain.hpp" + +extern "C" { +#include <signal.h> +} + +#include <atf-c++.hpp> + +#include "engine/config.hpp" +#include "engine/scheduler.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/env.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace scheduler = engine::scheduler; + +using utils::none; + + +namespace { + + +/// Copies the plain helper to the work directory, selecting a specific helper. +/// +/// \param tc Pointer to the calling test case, to obtain srcdir. +/// \param name Name of the new binary to create. Must match the name of a +/// valid helper, as the binary name is used to select it. +static void +copy_plain_helper(const atf::tests::tc* tc, const char* name) +{ + const fs::path srcdir(tc->get_config_var("srcdir")); + atf::utils::copy_file((srcdir / "plain_helpers").str(), name); +} + + +/// Runs one plain test program and checks its result. +/// +/// \param tc Pointer to the calling test case, to obtain srcdir. +/// \param test_case_name Name of the "test case" to select from the helper +/// program. +/// \param exp_result The expected result. +/// \param metadata The test case metadata. +/// \param user_config User-provided configuration variables. +static void +run_one(const atf::tests::tc* tc, const char* test_case_name, + const model::test_result& exp_result, + const model::metadata& metadata = model::metadata_builder().build(), + const config::tree& user_config = engine::empty_config()) +{ + copy_plain_helper(tc, test_case_name); + const model::test_program_ptr program = model::test_program_builder( + "plain", fs::path(test_case_name), fs::current_path(), "the-suite") + .add_test_case("main", metadata).build_ptr(); + + scheduler::scheduler_handle handle = scheduler::setup(); + (void)handle.spawn_test(program, "main", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: "); + atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: "); + ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(list); +ATF_TEST_CASE_BODY(list) +{ + const model::test_program program = model::test_program_builder( + "plain", fs::path("non-existent"), fs::path("."), "unused-suite") + .build(); + + scheduler::scheduler_handle handle = scheduler::setup(); + const model::test_cases_map test_cases = handle.list_tests( + &program, engine::empty_config()); + handle.cleanup(); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("main").build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__exit_success_is_pass); +ATF_TEST_CASE_BODY(test__exit_success_is_pass) +{ + const model::test_result exp_result(model::test_result_passed); + run_one(this, "pass", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__exit_non_zero_is_fail); +ATF_TEST_CASE_BODY(test__exit_non_zero_is_fail) +{ + const model::test_result exp_result( + model::test_result_failed, + "Returned non-success exit status 8"); + run_one(this, "fail", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__signal_is_broken); +ATF_TEST_CASE_BODY(test__signal_is_broken) +{ + const model::test_result exp_result(model::test_result_broken, + F("Received signal %s") % SIGABRT); + run_one(this, "crash", exp_result); +} + + +ATF_TEST_CASE(test__timeout_is_broken); +ATF_TEST_CASE_HEAD(test__timeout_is_broken) +{ + set_md_var("timeout", "60"); +} +ATF_TEST_CASE_BODY(test__timeout_is_broken) +{ + utils::setenv("CONTROL_DIR", fs::current_path().str()); + + const model::metadata metadata = model::metadata_builder() + .set_timeout(datetime::delta(1, 0)).build(); + const model::test_result exp_result(model::test_result_broken, + "Test case timed out"); + run_one(this, "timeout", exp_result, metadata); + + ATF_REQUIRE(!atf::utils::file_exists("cookie")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__configuration_variables); +ATF_TEST_CASE_BODY(test__configuration_variables) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.a-suite.first", "unused"); + user_config.set_string("test_suites.the-suite.first", "some value"); + user_config.set_string("test_suites.the-suite.second", "some other value"); + user_config.set_string("test_suites.other-suite.first", "unused"); + + const model::test_result exp_result(model::test_result_passed); + run_one(this, "check_configuration_variables", exp_result, + model::metadata_builder().build(), user_config); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + scheduler::register_interface( + "plain", std::shared_ptr< scheduler::interface >( + new engine::plain_interface())); + + ATF_ADD_TEST_CASE(tcs, list); + + ATF_ADD_TEST_CASE(tcs, test__exit_success_is_pass); + ATF_ADD_TEST_CASE(tcs, test__exit_non_zero_is_fail); + ATF_ADD_TEST_CASE(tcs, test__signal_is_broken); + ATF_ADD_TEST_CASE(tcs, test__timeout_is_broken); + ATF_ADD_TEST_CASE(tcs, test__configuration_variables); +} diff --git a/engine/requirements.cpp b/engine/requirements.cpp new file mode 100644 index 000000000000..a7b0a90d97db --- /dev/null +++ b/engine/requirements.cpp @@ -0,0 +1,293 @@ +// Copyright 2012 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/requirements.hpp" + +#include "model/metadata.hpp" +#include "model/types.hpp" +#include "utils/config/nodes.ipp" +#include "utils/config/tree.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/memory.hpp" +#include "utils/passwd.hpp" +#include "utils/sanity.hpp" +#include "utils/units.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace passwd = utils::passwd; +namespace units = utils::units; + + +namespace { + + +/// Checks if all required configuration variables are present. +/// +/// \param required_configs Set of required variable names. +/// \param user_config Runtime user configuration. +/// \param test_suite_name Name of the test suite the test belongs to. +/// +/// \return Empty if all variables are present or an error message otherwise. +static std::string +check_required_configs(const model::strings_set& required_configs, + const config::tree& user_config, + const std::string& test_suite_name) +{ + for (model::strings_set::const_iterator iter = required_configs.begin(); + iter != required_configs.end(); iter++) { + std::string property; + // TODO(jmmv): All this rewrite logic belongs in the ATF interface. + if ((*iter) == "unprivileged-user" || (*iter) == "unprivileged_user") + property = "unprivileged_user"; + else + property = F("test_suites.%s.%s") % test_suite_name % (*iter); + + if (!user_config.is_set(property)) + return F("Required configuration property '%s' not defined") % + (*iter); + } + return ""; +} + + +/// Checks if the allowed architectures match the current architecture. +/// +/// \param allowed_architectures Set of allowed architectures. +/// \param user_config Runtime user configuration. +/// +/// \return Empty if the current architecture is in the list or an error +/// message otherwise. +static std::string +check_allowed_architectures(const model::strings_set& allowed_architectures, + const config::tree& user_config) +{ + if (!allowed_architectures.empty()) { + const std::string architecture = + user_config.lookup< config::string_node >("architecture"); + if (allowed_architectures.find(architecture) == + allowed_architectures.end()) + return F("Current architecture '%s' not supported") % architecture; + } + return ""; +} + + +/// Checks if the allowed platforms match the current architecture. +/// +/// \param allowed_platforms Set of allowed platforms. +/// \param user_config Runtime user configuration. +/// +/// \return Empty if the current platform is in the list or an error message +/// otherwise. +static std::string +check_allowed_platforms(const model::strings_set& allowed_platforms, + const config::tree& user_config) +{ + if (!allowed_platforms.empty()) { + const std::string platform = + user_config.lookup< config::string_node >("platform"); + if (allowed_platforms.find(platform) == allowed_platforms.end()) + return F("Current platform '%s' not supported") % platform; + } + return ""; +} + + +/// Checks if the current user matches the required user. +/// +/// \param required_user Name of the required user category. +/// \param user_config Runtime user configuration. +/// +/// \return Empty if the current user fits the required user characteristics or +/// an error message otherwise. +static std::string +check_required_user(const std::string& required_user, + const config::tree& user_config) +{ + if (!required_user.empty()) { + const passwd::user user = passwd::current_user(); + if (required_user == "root") { + if (!user.is_root()) + return "Requires root privileges"; + } else if (required_user == "unprivileged") { + if (user.is_root()) + if (!user_config.is_set("unprivileged_user")) + return "Requires an unprivileged user but the " + "unprivileged-user configuration variable is not " + "defined"; + } else + UNREACHABLE_MSG("Value of require.user not properly validated"); + } + return ""; +} + + +/// Checks if all required files exist. +/// +/// \param required_files Set of paths. +/// +/// \return Empty if the required files all exist or an error message otherwise. +static std::string +check_required_files(const model::paths_set& required_files) +{ + for (model::paths_set::const_iterator iter = required_files.begin(); + iter != required_files.end(); iter++) { + INV((*iter).is_absolute()); + if (!fs::exists(*iter)) + return F("Required file '%s' not found") % *iter; + } + return ""; +} + + +/// Checks if all required programs exist. +/// +/// \param required_programs Set of paths. +/// +/// \return Empty if the required programs all exist or an error message +/// otherwise. +static std::string +check_required_programs(const model::paths_set& required_programs) +{ + for (model::paths_set::const_iterator iter = required_programs.begin(); + iter != required_programs.end(); iter++) { + if ((*iter).is_absolute()) { + if (!fs::exists(*iter)) + return F("Required program '%s' not found") % *iter; + } else { + if (!fs::find_in_path((*iter).c_str())) + return F("Required program '%s' not found in PATH") % *iter; + } + } + return ""; +} + + +/// Checks if the current system has the specified amount of memory. +/// +/// \param required_memory Amount of required physical memory, or zero if not +/// applicable. +/// +/// \return Empty if the current system has the required amount of memory or an +/// error message otherwise. +static std::string +check_required_memory(const units::bytes& required_memory) +{ + if (required_memory > 0) { + const units::bytes physical_memory = utils::physical_memory(); + if (physical_memory > 0 && physical_memory < required_memory) + return F("Requires %s bytes of physical memory but only %s " + "available") % + required_memory.format() % physical_memory.format(); + } + return ""; +} + + +/// Checks if the work directory's file system has enough free disk space. +/// +/// \param required_disk_space Amount of required free disk space, or zero if +/// not applicable. +/// \param work_directory Path to where the test case will be run. +/// +/// \return Empty if the file system where the work directory is hosted has +/// enough free disk space or an error message otherwise. +static std::string +check_required_disk_space(const units::bytes& required_disk_space, + const fs::path& work_directory) +{ + if (required_disk_space > 0) { + const units::bytes free_disk_space = fs::free_disk_space( + work_directory); + if (free_disk_space < required_disk_space) + return F("Requires %s bytes of free disk space but only %s " + "available") % + required_disk_space.format() % free_disk_space.format(); + } + return ""; +} + + +} // anonymous namespace + + +/// Checks if all the requirements specified by the test case are met. +/// +/// \param md The test metadata. +/// \param cfg The engine configuration. +/// \param test_suite Name of the test suite the test belongs to. +/// \param work_directory Path to where the test case will be run. +/// +/// \return A string describing the reason for skipping the test, or empty if +/// the test should be executed. +std::string +engine::check_reqs(const model::metadata& md, const config::tree& cfg, + const std::string& test_suite, + const fs::path& work_directory) +{ + std::string reason; + + reason = check_required_configs(md.required_configs(), cfg, test_suite); + if (!reason.empty()) + return reason; + + reason = check_allowed_architectures(md.allowed_architectures(), cfg); + if (!reason.empty()) + return reason; + + reason = check_allowed_platforms(md.allowed_platforms(), cfg); + if (!reason.empty()) + return reason; + + reason = check_required_user(md.required_user(), cfg); + if (!reason.empty()) + return reason; + + reason = check_required_files(md.required_files()); + if (!reason.empty()) + return reason; + + reason = check_required_programs(md.required_programs()); + if (!reason.empty()) + return reason; + + reason = check_required_memory(md.required_memory()); + if (!reason.empty()) + return reason; + + reason = check_required_disk_space(md.required_disk_space(), + work_directory); + if (!reason.empty()) + return reason; + + INV(reason.empty()); + return reason; +} diff --git a/engine/requirements.hpp b/engine/requirements.hpp new file mode 100644 index 000000000000..a36a938b3034 --- /dev/null +++ b/engine/requirements.hpp @@ -0,0 +1,51 @@ +// Copyright 2012 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/requirements.hpp +/// Handling of test case requirements. + +#if !defined(ENGINE_REQUIREMENTS_HPP) +#define ENGINE_REQUIREMENTS_HPP + +#include <string> + +#include "model/metadata_fwd.hpp" +#include "utils/config/tree_fwd.hpp" +#include "utils/fs/path_fwd.hpp" + +namespace engine { + + +std::string check_reqs(const model::metadata&, const utils::config::tree&, + const std::string&, const utils::fs::path&); + + +} // namespace engine + + +#endif // !defined(ENGINE_REQUIREMENTS_HPP) diff --git a/engine/requirements_test.cpp b/engine/requirements_test.cpp new file mode 100644 index 000000000000..5052da932cb6 --- /dev/null +++ b/engine/requirements_test.cpp @@ -0,0 +1,511 @@ +// Copyright 2012 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "model/metadata.hpp" + +#include <atf-c++.hpp> + +#include "engine/config.hpp" +#include "engine/requirements.hpp" +#include "utils/config/tree.ipp" +#include "utils/env.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/memory.hpp" +#include "utils/passwd.hpp" +#include "utils/units.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace passwd = utils::passwd; +namespace units = utils::units; + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__none); +ATF_TEST_CASE_BODY(check_reqs__none) +{ + const model::metadata md = model::metadata_builder().build(); + ATF_REQUIRE(engine::check_reqs(md, engine::empty_config(), "", + fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_architectures__one_ok); +ATF_TEST_CASE_BODY(check_reqs__allowed_architectures__one_ok) +{ + const model::metadata md = model::metadata_builder() + .add_allowed_architecture("x86_64") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("architecture", "x86_64"); + user_config.set_string("platform", ""); + ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_architectures__one_fail); +ATF_TEST_CASE_BODY(check_reqs__allowed_architectures__one_fail) +{ + const model::metadata md = model::metadata_builder() + .add_allowed_architecture("x86_64") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("architecture", "i386"); + user_config.set_string("platform", ""); + ATF_REQUIRE_MATCH("Current architecture 'i386' not supported", + engine::check_reqs(md, user_config, "", fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_architectures__many_ok); +ATF_TEST_CASE_BODY(check_reqs__allowed_architectures__many_ok) +{ + const model::metadata md = model::metadata_builder() + .add_allowed_architecture("x86_64") + .add_allowed_architecture("i386") + .add_allowed_architecture("powerpc") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("architecture", "i386"); + user_config.set_string("platform", ""); + ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_architectures__many_fail); +ATF_TEST_CASE_BODY(check_reqs__allowed_architectures__many_fail) +{ + const model::metadata md = model::metadata_builder() + .add_allowed_architecture("x86_64") + .add_allowed_architecture("i386") + .add_allowed_architecture("powerpc") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("architecture", "arm"); + user_config.set_string("platform", ""); + ATF_REQUIRE_MATCH("Current architecture 'arm' not supported", + engine::check_reqs(md, user_config, "", fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_platforms__one_ok); +ATF_TEST_CASE_BODY(check_reqs__allowed_platforms__one_ok) +{ + const model::metadata md = model::metadata_builder() + .add_allowed_platform("amd64") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("architecture", ""); + user_config.set_string("platform", "amd64"); + ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_platforms__one_fail); +ATF_TEST_CASE_BODY(check_reqs__allowed_platforms__one_fail) +{ + const model::metadata md = model::metadata_builder() + .add_allowed_platform("amd64") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("architecture", ""); + user_config.set_string("platform", "i386"); + ATF_REQUIRE_MATCH("Current platform 'i386' not supported", + engine::check_reqs(md, user_config, "", fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_platforms__many_ok); +ATF_TEST_CASE_BODY(check_reqs__allowed_platforms__many_ok) +{ + const model::metadata md = model::metadata_builder() + .add_allowed_platform("amd64") + .add_allowed_platform("i386") + .add_allowed_platform("macppc") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("architecture", ""); + user_config.set_string("platform", "i386"); + ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__allowed_platforms__many_fail); +ATF_TEST_CASE_BODY(check_reqs__allowed_platforms__many_fail) +{ + const model::metadata md = model::metadata_builder() + .add_allowed_platform("amd64") + .add_allowed_platform("i386") + .add_allowed_platform("macppc") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("architecture", ""); + user_config.set_string("platform", "shark"); + ATF_REQUIRE_MATCH("Current platform 'shark' not supported", + engine::check_reqs(md, user_config, "", fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_configs__one_ok); +ATF_TEST_CASE_BODY(check_reqs__required_configs__one_ok) +{ + const model::metadata md = model::metadata_builder() + .add_required_config("my-var") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("test_suites.suite.aaa", "value1"); + user_config.set_string("test_suites.suite.my-var", "value2"); + user_config.set_string("test_suites.suite.zzz", "value3"); + ATF_REQUIRE(engine::check_reqs(md, user_config, "suite", + fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_configs__one_fail); +ATF_TEST_CASE_BODY(check_reqs__required_configs__one_fail) +{ + const model::metadata md = model::metadata_builder() + .add_required_config("unprivileged_user") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("test_suites.suite.aaa", "value1"); + user_config.set_string("test_suites.suite.my-var", "value2"); + user_config.set_string("test_suites.suite.zzz", "value3"); + ATF_REQUIRE_MATCH("Required configuration property 'unprivileged_user' not " + "defined", + engine::check_reqs(md, user_config, "suite", + fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_configs__many_ok); +ATF_TEST_CASE_BODY(check_reqs__required_configs__many_ok) +{ + const model::metadata md = model::metadata_builder() + .add_required_config("foo") + .add_required_config("bar") + .add_required_config("baz") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("test_suites.suite.aaa", "value1"); + user_config.set_string("test_suites.suite.foo", "value2"); + user_config.set_string("test_suites.suite.bar", "value3"); + user_config.set_string("test_suites.suite.baz", "value4"); + user_config.set_string("test_suites.suite.zzz", "value5"); + ATF_REQUIRE(engine::check_reqs(md, user_config, "suite", + fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_configs__many_fail); +ATF_TEST_CASE_BODY(check_reqs__required_configs__many_fail) +{ + const model::metadata md = model::metadata_builder() + .add_required_config("foo") + .add_required_config("bar") + .add_required_config("baz") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set_string("test_suites.suite.aaa", "value1"); + user_config.set_string("test_suites.suite.foo", "value2"); + user_config.set_string("test_suites.suite.zzz", "value3"); + ATF_REQUIRE_MATCH("Required configuration property 'bar' not defined", + engine::check_reqs(md, user_config, "suite", + fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_configs__special); +ATF_TEST_CASE_BODY(check_reqs__required_configs__special) +{ + const model::metadata md = model::metadata_builder() + .add_required_config("unprivileged-user") + .build(); + + config::tree user_config = engine::default_config(); + ATF_REQUIRE_MATCH("Required configuration property 'unprivileged-user' " + "not defined", + engine::check_reqs(md, user_config, "", fs::path("."))); + user_config.set< engine::user_node >( + "unprivileged_user", passwd::user("foo", 1, 2)); + ATF_REQUIRE(engine::check_reqs(md, user_config, "foo", + fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_user__root__ok); +ATF_TEST_CASE_BODY(check_reqs__required_user__root__ok) +{ + const model::metadata md = model::metadata_builder() + .set_required_user("root") + .build(); + + config::tree user_config = engine::default_config(); + ATF_REQUIRE(!user_config.is_set("unprivileged_user")); + + passwd::set_current_user_for_testing(passwd::user("", 0, 1)); + ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_user__root__fail); +ATF_TEST_CASE_BODY(check_reqs__required_user__root__fail) +{ + const model::metadata md = model::metadata_builder() + .set_required_user("root") + .build(); + + passwd::set_current_user_for_testing(passwd::user("", 123, 1)); + ATF_REQUIRE_MATCH("Requires root privileges", + engine::check_reqs(md, engine::empty_config(), "", + fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_user__unprivileged__same); +ATF_TEST_CASE_BODY(check_reqs__required_user__unprivileged__same) +{ + const model::metadata md = model::metadata_builder() + .set_required_user("unprivileged") + .build(); + + config::tree user_config = engine::default_config(); + ATF_REQUIRE(!user_config.is_set("unprivileged_user")); + + passwd::set_current_user_for_testing(passwd::user("", 123, 1)); + ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_user__unprivileged__ok); +ATF_TEST_CASE_BODY(check_reqs__required_user__unprivileged__ok) +{ + const model::metadata md = model::metadata_builder() + .set_required_user("unprivileged") + .build(); + + config::tree user_config = engine::default_config(); + user_config.set< engine::user_node >( + "unprivileged_user", passwd::user("", 123, 1)); + + passwd::set_current_user_for_testing(passwd::user("", 0, 1)); + ATF_REQUIRE(engine::check_reqs(md, user_config, "", fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_user__unprivileged__fail); +ATF_TEST_CASE_BODY(check_reqs__required_user__unprivileged__fail) +{ + const model::metadata md = model::metadata_builder() + .set_required_user("unprivileged") + .build(); + + config::tree user_config = engine::default_config(); + ATF_REQUIRE(!user_config.is_set("unprivileged_user")); + + passwd::set_current_user_for_testing(passwd::user("", 0, 1)); + ATF_REQUIRE_MATCH("Requires.*unprivileged.*unprivileged-user", + engine::check_reqs(md, user_config, "", fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_disk_space__ok); +ATF_TEST_CASE_BODY(check_reqs__required_disk_space__ok) +{ + const model::metadata md = model::metadata_builder() + .set_required_disk_space(units::bytes::parse("1m")) + .build(); + + ATF_REQUIRE(engine::check_reqs(md, engine::empty_config(), "", + fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_disk_space__fail); +ATF_TEST_CASE_BODY(check_reqs__required_disk_space__fail) +{ + const model::metadata md = model::metadata_builder() + .set_required_disk_space(units::bytes::parse("1000t")) + .build(); + + ATF_REQUIRE_MATCH("Requires 1000.00T .*disk space", + engine::check_reqs(md, engine::empty_config(), "", + fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_files__ok); +ATF_TEST_CASE_BODY(check_reqs__required_files__ok) +{ + const model::metadata md = model::metadata_builder() + .add_required_file(fs::current_path() / "test-file") + .build(); + + atf::utils::create_file("test-file", ""); + + ATF_REQUIRE(engine::check_reqs(md, engine::empty_config(), "", + fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_files__fail); +ATF_TEST_CASE_BODY(check_reqs__required_files__fail) +{ + const model::metadata md = model::metadata_builder() + .add_required_file(fs::path("/non-existent/file")) + .build(); + + ATF_REQUIRE_MATCH("'/non-existent/file' not found$", + engine::check_reqs(md, engine::empty_config(), "", + fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_memory__ok); +ATF_TEST_CASE_BODY(check_reqs__required_memory__ok) +{ + const model::metadata md = model::metadata_builder() + .set_required_memory(units::bytes::parse("1m")) + .build(); + + ATF_REQUIRE(engine::check_reqs(md, engine::empty_config(), "", + fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_memory__fail); +ATF_TEST_CASE_BODY(check_reqs__required_memory__fail) +{ + const model::metadata md = model::metadata_builder() + .set_required_memory(units::bytes::parse("100t")) + .build(); + + if (utils::physical_memory() == 0) + skip("Don't know how to query the amount of physical memory"); + ATF_REQUIRE_MATCH("Requires 100.00T .*memory", + engine::check_reqs(md, engine::empty_config(), "", + fs::path("."))); +} + + +ATF_TEST_CASE(check_reqs__required_programs__ok); +ATF_TEST_CASE_HEAD(check_reqs__required_programs__ok) +{ + set_md_var("require.progs", "/bin/ls /bin/mv"); +} +ATF_TEST_CASE_BODY(check_reqs__required_programs__ok) +{ + const model::metadata md = model::metadata_builder() + .add_required_program(fs::path("/bin/ls")) + .add_required_program(fs::path("foo")) + .add_required_program(fs::path("/bin/mv")) + .build(); + + fs::mkdir(fs::path("bin"), 0755); + atf::utils::create_file("bin/foo", ""); + utils::setenv("PATH", (fs::current_path() / "bin").str()); + + ATF_REQUIRE(engine::check_reqs(md, engine::empty_config(), "", + fs::path(".")).empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_programs__fail_absolute); +ATF_TEST_CASE_BODY(check_reqs__required_programs__fail_absolute) +{ + const model::metadata md = model::metadata_builder() + .add_required_program(fs::path("/non-existent/program")) + .build(); + + ATF_REQUIRE_MATCH("'/non-existent/program' not found$", + engine::check_reqs(md, engine::empty_config(), "", + fs::path("."))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(check_reqs__required_programs__fail_relative); +ATF_TEST_CASE_BODY(check_reqs__required_programs__fail_relative) +{ + const model::metadata md = model::metadata_builder() + .add_required_program(fs::path("foo")) + .add_required_program(fs::path("bar")) + .build(); + + fs::mkdir(fs::path("bin"), 0755); + atf::utils::create_file("bin/foo", ""); + utils::setenv("PATH", (fs::current_path() / "bin").str()); + + ATF_REQUIRE_MATCH("'bar' not found in PATH$", + engine::check_reqs(md, engine::empty_config(), "", + fs::path("."))); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, check_reqs__none); + ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_architectures__one_ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_architectures__one_fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_architectures__many_ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_architectures__many_fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_platforms__one_ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_platforms__one_fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_platforms__many_ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__allowed_platforms__many_fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_configs__one_ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_configs__one_fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_configs__many_ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_configs__many_fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_configs__special); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_user__root__ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_user__root__fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_user__unprivileged__same); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_user__unprivileged__ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_user__unprivileged__fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_disk_space__ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_disk_space__fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_files__ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_files__fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_memory__ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_memory__fail); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_programs__ok); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_programs__fail_absolute); + ATF_ADD_TEST_CASE(tcs, check_reqs__required_programs__fail_relative); +} diff --git a/engine/scanner.cpp b/engine/scanner.cpp new file mode 100644 index 000000000000..b42b089c3c3c --- /dev/null +++ b/engine/scanner.cpp @@ -0,0 +1,216 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/scanner.hpp" + +#include <deque> +#include <string> + +#include "engine/filters.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "utils/noncopyable.hpp" +#include "utils/optional.ipp" +#include "utils/sanity.hpp" + +using utils::none; +using utils::optional; + + +namespace { + + +/// Extracts the keys of a map as a deque. +/// +/// \tparam KeyType The type of the map keys. +/// \tparam ValueType The type of the map values. +/// \param map The input map. +/// +/// \return A deque with the keys of the map. +template< typename KeyType, typename ValueType > +static std::deque< KeyType > +map_keys(const std::map< KeyType, ValueType >& map) +{ + std::deque< KeyType > keys; + for (typename std::map< KeyType, ValueType >::const_iterator iter = + map.begin(); iter != map.end(); ++iter) { + keys.push_back((*iter).first); + } + return keys; +} + + +} // anonymous namespace + + +/// Internal implementation for the scanner class. +struct engine::scanner::impl : utils::noncopyable { + /// Collection of test programs not yet scanned. + /// + /// The first element in this deque is the "active" test program when + /// first_test_cases is defined. + std::deque< model::test_program_ptr > pending_test_programs; + + /// Current state of the provided filters. + engine::filters_state filters; + + /// Collection of test cases not yet scanned. + /// + /// These are the test cases for the first test program in + /// pending_test_programs when such test program is active. + optional< std::deque< std::string > > first_test_cases; + + /// Constructor. + /// + /// \param test_programs_ Collection of test programs to scan through. + /// \param filters_ List of scan filters as provided by the user. + impl(const model::test_programs_vector& test_programs_, + const std::set< engine::test_filter >& filters_) : + pending_test_programs(test_programs_.begin(), test_programs_.end()), + filters(filters_) + { + } + + /// Positions the internal state to return the next element if any. + /// + /// \post If there are more elements to read, returns true and + /// pending_test_programs[0] points to the active test program and + /// first_test_cases[0] has the test case to be returned. + /// + /// \return True if there is one more result available. + bool + advance(void) + { + for (;;) { + if (first_test_cases) { + if (first_test_cases.get().empty()) { + pending_test_programs.pop_front(); + first_test_cases = none; + } + } + if (pending_test_programs.empty()) { + break; + } + + model::test_program_ptr test_program = pending_test_programs[0]; + if (!first_test_cases) { + if (!filters.match_test_program( + test_program->relative_path())) { + pending_test_programs.pop_front(); + continue; + } + + first_test_cases = utils::make_optional( + map_keys(test_program->test_cases())); + } + + if (!first_test_cases.get().empty()) { + std::deque< std::string >::iterator iter = + first_test_cases.get().begin(); + const std::string test_case_name = *iter; + if (!filters.match_test_case(test_program->relative_path(), + test_case_name)) { + first_test_cases.get().erase(iter); + continue; + } + return true; + } else { + pending_test_programs.pop_front(); + first_test_cases = none; + } + } + return false; + } + + /// Extracts the current element. + /// + /// \pre Must be called only if advance() returns true, and immediately + /// afterwards. + /// + /// \return The current scan result. + engine::scan_result + consume(void) + { + const std::string test_case_name = first_test_cases.get()[0]; + first_test_cases.get().pop_front(); + return scan_result(pending_test_programs[0], test_case_name); + } +}; + + +/// Constructor. +/// +/// \param test_programs Collection of test programs to scan through. +/// \param filters List of scan filters as provided by the user. +engine::scanner::scanner(const model::test_programs_vector& test_programs, + const std::set< engine::test_filter >& filters) : + _pimpl(new impl(test_programs, filters)) +{ +} + + +/// Destructor. +engine::scanner::~scanner(void) +{ +} + + +/// Returns the next scan result. +/// +/// \return A scan result if there are still pending test cases to be processed, +/// or none otherwise. +optional< engine::scan_result > +engine::scanner::yield(void) +{ + if (_pimpl->advance()) { + return utils::make_optional(_pimpl->consume()); + } else { + return none; + } +} + + +/// Checks whether the scan is finished. +/// +/// \return True if the scan is finished, in which case yield() will return +/// none; false otherwise. +bool +engine::scanner::done(void) +{ + return !_pimpl->advance(); +} + + +/// Returns the list of test filters that did not match any test case. +/// +/// \return The collection of unmatched test filters. +std::set< engine::test_filter > +engine::scanner::unused_filters(void) const +{ + return _pimpl->filters.unused(); +} diff --git a/engine/scanner.hpp b/engine/scanner.hpp new file mode 100644 index 000000000000..722bc9be5f4c --- /dev/null +++ b/engine/scanner.hpp @@ -0,0 +1,76 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/scanner.hpp +/// Utilities to scan through list of tests in a test suite. + +#if !defined(ENGINE_SCANNER_HPP) +#define ENGINE_SCANNER_HPP + +#include "engine/scanner_fwd.hpp" + +#include <memory> +#include <set> + +#include "engine/filters_fwd.hpp" +#include "model/test_program_fwd.hpp" +#include "utils/optional_fwd.hpp" + +namespace engine { + + +/// Scans a list of test programs, yielding one test case at a time. +/// +/// This class contains the state necessary to process a collection of test +/// programs (possibly as provided by the Kyuafile) and to extract an arbitrary +/// (test program, test_case) pair out of them one at a time. +/// +/// The scanning algorithm guarantees that test programs are initialized +/// dynamically, should they need to load their list of test cases from disk. +/// +/// The order of the extraction is not guaranteed. +class scanner { + struct impl; + /// Pointer to the internal implementation data. + std::shared_ptr< impl > _pimpl; + +public: + scanner(const model::test_programs_vector&, const std::set< test_filter >&); + ~scanner(void); + + bool done(void); + utils::optional< scan_result > yield(void); + + std::set< test_filter > unused_filters(void) const; +}; + + +} // namespace engine + + +#endif // !defined(ENGINE_SCANNER_HPP) diff --git a/engine/scanner_fwd.hpp b/engine/scanner_fwd.hpp new file mode 100644 index 000000000000..5c91888fa266 --- /dev/null +++ b/engine/scanner_fwd.hpp @@ -0,0 +1,59 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/scanner_fwd.hpp +/// Forward declarations for engine/scanner.hpp + +#if !defined(ENGINE_SCANNER_FWD_HPP) +#define ENGINE_SCANNER_FWD_HPP + +#include <string> +#include <utility> + +#include "model/test_program_fwd.hpp" + +namespace engine { + + +/// Result type yielded by the scanner: a (test program, test case name) pair. +/// +/// We must use model::test_program_ptr here instead of model::test_program +/// because we must keep the polimorphic properties of the test program. In +/// particular, if the test program comes from the Kyuafile and is of the type +/// model::lazy_test_program, we must keep access to the loaded list of test +/// cases (which, for obscure reasons, is kept in the subclass). +/// TODO(jmmv): This is ugly, very ugly. There has to be a better way. +typedef std::pair< model::test_program_ptr, std::string > scan_result; + + +class scanner; + + +} // namespace engine + +#endif // !defined(ENGINE_SCANNER_FWD_HPP) diff --git a/engine/scanner_test.cpp b/engine/scanner_test.cpp new file mode 100644 index 000000000000..f79717eca49e --- /dev/null +++ b/engine/scanner_test.cpp @@ -0,0 +1,476 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/scanner.hpp" + +#include <cstdarg> +#include <cstddef> +#include <typeinfo> + +#include <atf-c++.hpp> + +#include "engine/filters.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "utils/format/containers.ipp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" + +namespace fs = utils::fs; + +using utils::optional; + + +namespace { + + +/// Test program that implements a mock test_cases() lazy call. +class mock_test_program : public model::test_program { + /// Number of times test_cases has been called. + mutable std::size_t _num_calls; + + /// Collection of test cases; lazily initialized. + mutable model::test_cases_map _test_cases; + +public: + /// Constructs a new test program. + /// + /// \param binary_ The name of the test program binary relative to root_. + mock_test_program(const fs::path& binary_) : + test_program("unused-interface", binary_, fs::path("unused-root"), + "unused-suite", model::metadata_builder().build(), + model::test_cases_map()), + _num_calls(0) + { + } + + /// Gets or loads the list of test cases from the test program. + /// + /// \return The list of test cases provided by the test program. + const model::test_cases_map& + test_cases(void) const + { + if (_num_calls == 0) { + const model::metadata metadata = model::metadata_builder().build(); + const model::test_case tc1("one", metadata); + const model::test_case tc2("two", metadata); + _test_cases.insert(model::test_cases_map::value_type("one", tc1)); + _test_cases.insert(model::test_cases_map::value_type("two", tc2)); + } + _num_calls++; + return _test_cases; + } + + /// Returns the number of times test_cases() has been called. + /// + /// \return A counter. + std::size_t + num_calls(void) const + { + return _num_calls; + } +}; + + +/// Syntactic sugar to instantiate a test program with various test cases. +/// +/// The scanner only cares about the relative path of the test program object +/// and the names of the test cases. This function helps in instantiating a +/// test program that has the minimum set of details only. +/// +/// \param relative_path Relative path to the test program. +/// \param ... List of test case names to add to the test program. Must be +/// NULL-terminated. +/// +/// \return A constructed test program. +static model::test_program_ptr +new_test_program(const char* relative_path, ...) +{ + model::test_program_builder builder( + "unused-interface", fs::path(relative_path), fs::path("unused-root"), + "unused-suite"); + + va_list ap; + va_start(ap, relative_path); + const char* test_case_name; + while ((test_case_name = va_arg(ap, const char*)) != NULL) { + builder.add_test_case(test_case_name); + } + va_end(ap); + + return builder.build_ptr(); +} + + +/// Yields all test cases in the scanner for simplicity of testing. +/// +/// In most of the tests below, we just care about the scanner returning the +/// full set of matching test cases, not the specific behavior of every single +/// yield() call. This function just returns the whole set, which helps in +/// writing functional tests. +/// +/// \param scanner The scanner on which to iterate. +/// +/// \return The full collection of results yielded by the scanner. +static std::set< engine::scan_result > +yield_all(engine::scanner& scanner) +{ + std::set< engine::scan_result > results; + while (!scanner.done()) { + const optional< engine::scan_result > result = scanner.yield(); + ATF_REQUIRE(result); + results.insert(result.get()); + } + ATF_REQUIRE(!scanner.yield()); + ATF_REQUIRE(scanner.done()); + return results; +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__no_tests); +ATF_TEST_CASE_BODY(scanner__no_filters__no_tests) +{ + const model::test_programs_vector test_programs; + const std::set< engine::test_filter > filters; + + engine::scanner scanner(test_programs, filters); + ATF_REQUIRE(scanner.done()); + ATF_REQUIRE(!scanner.yield()); + ATF_REQUIRE(scanner.unused_filters().empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__one_test_in_one_program); +ATF_TEST_CASE_BODY(scanner__no_filters__one_test_in_one_program) +{ + const model::test_program_ptr test_program = new_test_program( + "dir/program", "lone_test", NULL); + + model::test_programs_vector test_programs; + test_programs.push_back(test_program); + + const std::set< engine::test_filter > filters; + + std::set< engine::scan_result > exp_results; + exp_results.insert(engine::scan_result(test_program, "lone_test")); + + engine::scanner scanner(test_programs, filters); + const std::set< engine::scan_result > results = yield_all(scanner); + ATF_REQUIRE_EQ(exp_results, results); + ATF_REQUIRE(scanner.unused_filters().empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__one_test_per_many_programs); +ATF_TEST_CASE_BODY(scanner__no_filters__one_test_per_many_programs) +{ + const model::test_program_ptr test_program1 = new_test_program( + "dir/program1", "foo_test", NULL); + const model::test_program_ptr test_program2 = new_test_program( + "program2", "bar_test", NULL); + const model::test_program_ptr test_program3 = new_test_program( + "a/b/c/d/e/program3", "baz_test", NULL); + + model::test_programs_vector test_programs; + test_programs.push_back(test_program1); + test_programs.push_back(test_program2); + test_programs.push_back(test_program3); + + const std::set< engine::test_filter > filters; + + std::set< engine::scan_result > exp_results; + exp_results.insert(engine::scan_result(test_program1, "foo_test")); + exp_results.insert(engine::scan_result(test_program2, "bar_test")); + exp_results.insert(engine::scan_result(test_program3, "baz_test")); + + engine::scanner scanner(test_programs, filters); + const std::set< engine::scan_result > results = yield_all(scanner); + ATF_REQUIRE_EQ(exp_results, results); + ATF_REQUIRE(scanner.unused_filters().empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__many_tests_in_one_program); +ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_in_one_program) +{ + const model::test_program_ptr test_program = new_test_program( + "dir/program", "first_test", "second_test", "third_test", NULL); + + model::test_programs_vector test_programs; + test_programs.push_back(test_program); + + const std::set< engine::test_filter > filters; + + std::set< engine::scan_result > exp_results; + exp_results.insert(engine::scan_result(test_program, "first_test")); + exp_results.insert(engine::scan_result(test_program, "second_test")); + exp_results.insert(engine::scan_result(test_program, "third_test")); + + engine::scanner scanner(test_programs, filters); + const std::set< engine::scan_result > results = yield_all(scanner); + ATF_REQUIRE_EQ(exp_results, results); + ATF_REQUIRE(scanner.unused_filters().empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__many_tests_per_many_programs); +ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_per_many_programs) +{ + const model::test_program_ptr test_program1 = new_test_program( + "dir/program1", "foo_test", "bar_test", "baz_test", NULL); + const model::test_program_ptr test_program2 = new_test_program( + "program2", "lone_test", NULL); + const model::test_program_ptr test_program3 = new_test_program( + "a/b/c/d/e/program3", "another_test", "last_test", NULL); + + model::test_programs_vector test_programs; + test_programs.push_back(test_program1); + test_programs.push_back(test_program2); + test_programs.push_back(test_program3); + + const std::set< engine::test_filter > filters; + + std::set< engine::scan_result > exp_results; + exp_results.insert(engine::scan_result(test_program1, "foo_test")); + exp_results.insert(engine::scan_result(test_program1, "bar_test")); + exp_results.insert(engine::scan_result(test_program1, "baz_test")); + exp_results.insert(engine::scan_result(test_program2, "lone_test")); + exp_results.insert(engine::scan_result(test_program3, "another_test")); + exp_results.insert(engine::scan_result(test_program3, "last_test")); + + engine::scanner scanner(test_programs, filters); + const std::set< engine::scan_result > results = yield_all(scanner); + ATF_REQUIRE_EQ(exp_results, results); + ATF_REQUIRE(scanner.unused_filters().empty()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__verify_lazy_loads); +ATF_TEST_CASE_BODY(scanner__no_filters__verify_lazy_loads) +{ + const model::test_program_ptr test_program1(new mock_test_program( + fs::path("first"))); + const mock_test_program* mock_program1 = + dynamic_cast< const mock_test_program* >(test_program1.get()); + const model::test_program_ptr test_program2(new mock_test_program( + fs::path("second"))); + const mock_test_program* mock_program2 = + dynamic_cast< const mock_test_program* >(test_program2.get()); + + model::test_programs_vector test_programs; + test_programs.push_back(test_program1); + test_programs.push_back(test_program2); + + const std::set< engine::test_filter > filters; + + std::set< engine::scan_result > exp_results; + exp_results.insert(engine::scan_result(test_program1, "one")); + exp_results.insert(engine::scan_result(test_program1, "two")); + exp_results.insert(engine::scan_result(test_program2, "one")); + exp_results.insert(engine::scan_result(test_program2, "two")); + + engine::scanner scanner(test_programs, filters); + std::set< engine::scan_result > results; + ATF_REQUIRE_EQ(0, mock_program1->num_calls()); + ATF_REQUIRE_EQ(0, mock_program2->num_calls()); + + // This abuses the internal implementation of the scanner by making + // assumptions on the order of the results. + results.insert(scanner.yield().get()); + ATF_REQUIRE_EQ(1, mock_program1->num_calls()); + ATF_REQUIRE_EQ(0, mock_program2->num_calls()); + results.insert(scanner.yield().get()); + ATF_REQUIRE_EQ(1, mock_program1->num_calls()); + ATF_REQUIRE_EQ(0, mock_program2->num_calls()); + results.insert(scanner.yield().get()); + ATF_REQUIRE_EQ(1, mock_program1->num_calls()); + ATF_REQUIRE_EQ(1, mock_program2->num_calls()); + results.insert(scanner.yield().get()); + ATF_REQUIRE_EQ(1, mock_program1->num_calls()); + ATF_REQUIRE_EQ(1, mock_program2->num_calls()); + ATF_REQUIRE(scanner.done()); + + ATF_REQUIRE_EQ(exp_results, results); + ATF_REQUIRE(scanner.unused_filters().empty()); + + // Make sure we are still talking to the original objects. + for (std::set< engine::scan_result >::const_iterator iter = results.begin(); + iter != results.end(); ++iter) { + const mock_test_program* mock_program = + dynamic_cast< const mock_test_program* >((*iter).first.get()); + ATF_REQUIRE_EQ(1, mock_program->num_calls()); + } +} + + +ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__no_tests); +ATF_TEST_CASE_BODY(scanner__with_filters__no_tests) +{ + const model::test_programs_vector test_programs; + + std::set< engine::test_filter > filters; + filters.insert(engine::test_filter(fs::path("foo"), "bar")); + + engine::scanner scanner(test_programs, filters); + ATF_REQUIRE(scanner.done()); + ATF_REQUIRE(!scanner.yield()); + ATF_REQUIRE_EQ(filters, scanner.unused_filters()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__no_matches); +ATF_TEST_CASE_BODY(scanner__with_filters__no_matches) +{ + const model::test_program_ptr test_program1 = new_test_program( + "dir/program1", "foo_test", "bar_test", "baz_test", NULL); + const model::test_program_ptr test_program2 = new_test_program( + "dir/program2", "bar_test", NULL); + const model::test_program_ptr test_program3 = new_test_program( + "program3", "another_test", "last_test", NULL); + + model::test_programs_vector test_programs; + test_programs.push_back(test_program1); + test_programs.push_back(test_program2); + test_programs.push_back(test_program3); + + std::set< engine::test_filter > filters; + filters.insert(engine::test_filter(fs::path("dir/program2"), "baz_test")); + filters.insert(engine::test_filter(fs::path("program4"), "another_test")); + filters.insert(engine::test_filter(fs::path("dir/program3"), "")); + + const std::set< engine::scan_result > exp_results; + + engine::scanner scanner(test_programs, filters); + const std::set< engine::scan_result > results = yield_all(scanner); + ATF_REQUIRE_EQ(exp_results, results); + ATF_REQUIRE_EQ(filters, scanner.unused_filters()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__some_matches); +ATF_TEST_CASE_BODY(scanner__with_filters__some_matches) +{ + const model::test_program_ptr test_program1 = new_test_program( + "dir/program1", "foo_test", "bar_test", "baz_test", NULL); + const model::test_program_ptr test_program2 = new_test_program( + "dir/program2", "bar_test", NULL); + const model::test_program_ptr test_program3 = new_test_program( + "program3", "another_test", "last_test", NULL); + const model::test_program_ptr test_program4 = new_test_program( + "program4", "more_test", NULL); + + model::test_programs_vector test_programs; + test_programs.push_back(test_program1); + test_programs.push_back(test_program2); + test_programs.push_back(test_program3); + test_programs.push_back(test_program4); + + std::set< engine::test_filter > filters; + filters.insert(engine::test_filter(fs::path("dir/program1"), "baz_test")); + filters.insert(engine::test_filter(fs::path("dir/program2"), "foo_test")); + filters.insert(engine::test_filter(fs::path("program3"), "")); + + std::set< engine::test_filter > exp_filters; + exp_filters.insert(engine::test_filter(fs::path("dir/program2"), + "foo_test")); + + std::set< engine::scan_result > exp_results; + exp_results.insert(engine::scan_result(test_program1, "baz_test")); + exp_results.insert(engine::scan_result(test_program3, "another_test")); + exp_results.insert(engine::scan_result(test_program3, "last_test")); + + engine::scanner scanner(test_programs, filters); + const std::set< engine::scan_result > results = yield_all(scanner); + ATF_REQUIRE_EQ(exp_results, results); + + ATF_REQUIRE_EQ(exp_filters, scanner.unused_filters()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__verify_lazy_loads); +ATF_TEST_CASE_BODY(scanner__with_filters__verify_lazy_loads) +{ + const model::test_program_ptr test_program1(new mock_test_program( + fs::path("first"))); + const mock_test_program* mock_program1 = + dynamic_cast< const mock_test_program* >(test_program1.get()); + const model::test_program_ptr test_program2(new mock_test_program( + fs::path("second"))); + const mock_test_program* mock_program2 = + dynamic_cast< const mock_test_program* >(test_program2.get()); + + model::test_programs_vector test_programs; + test_programs.push_back(test_program1); + test_programs.push_back(test_program2); + + std::set< engine::test_filter > filters; + filters.insert(engine::test_filter(fs::path("first"), "")); + + std::set< engine::scan_result > exp_results; + exp_results.insert(engine::scan_result(test_program1, "one")); + exp_results.insert(engine::scan_result(test_program1, "two")); + + engine::scanner scanner(test_programs, filters); + std::set< engine::scan_result > results; + ATF_REQUIRE_EQ(0, mock_program1->num_calls()); + ATF_REQUIRE_EQ(0, mock_program2->num_calls()); + + results.insert(scanner.yield().get()); + ATF_REQUIRE_EQ(1, mock_program1->num_calls()); + ATF_REQUIRE_EQ(0, mock_program2->num_calls()); + results.insert(scanner.yield().get()); + ATF_REQUIRE_EQ(1, mock_program1->num_calls()); + ATF_REQUIRE_EQ(0, mock_program2->num_calls()); + ATF_REQUIRE(scanner.done()); + + ATF_REQUIRE_EQ(exp_results, results); + ATF_REQUIRE(scanner.unused_filters().empty()); + + ATF_REQUIRE_EQ(1, mock_program1->num_calls()); + ATF_REQUIRE_EQ(0, mock_program2->num_calls()); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, scanner__no_filters__no_tests); + ATF_ADD_TEST_CASE(tcs, scanner__no_filters__one_test_in_one_program); + ATF_ADD_TEST_CASE(tcs, scanner__no_filters__one_test_per_many_programs); + ATF_ADD_TEST_CASE(tcs, scanner__no_filters__many_tests_in_one_program); + ATF_ADD_TEST_CASE(tcs, scanner__no_filters__many_tests_per_many_programs); + ATF_ADD_TEST_CASE(tcs, scanner__no_filters__verify_lazy_loads); + + ATF_ADD_TEST_CASE(tcs, scanner__with_filters__no_tests); + ATF_ADD_TEST_CASE(tcs, scanner__with_filters__no_matches); + ATF_ADD_TEST_CASE(tcs, scanner__with_filters__some_matches); + ATF_ADD_TEST_CASE(tcs, scanner__with_filters__verify_lazy_loads); +} diff --git a/engine/scheduler.cpp b/engine/scheduler.cpp new file mode 100644 index 000000000000..e7b51d23acca --- /dev/null +++ b/engine/scheduler.cpp @@ -0,0 +1,1373 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/scheduler.hpp" + +extern "C" { +#include <unistd.h> +} + +#include <cstdio> +#include <cstdlib> +#include <fstream> +#include <memory> +#include <stdexcept> + +#include "engine/config.hpp" +#include "engine/exceptions.hpp" +#include "engine/requirements.hpp" +#include "model/context.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/defs.hpp" +#include "utils/env.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/directory.hpp" +#include "utils/fs/exceptions.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/logging/macros.hpp" +#include "utils/noncopyable.hpp" +#include "utils/optional.ipp" +#include "utils/passwd.hpp" +#include "utils/process/executor.ipp" +#include "utils/process/status.hpp" +#include "utils/sanity.hpp" +#include "utils/stacktrace.hpp" +#include "utils/stream.hpp" +#include "utils/text/operations.ipp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace executor = utils::process::executor; +namespace fs = utils::fs; +namespace logging = utils::logging; +namespace passwd = utils::passwd; +namespace process = utils::process; +namespace scheduler = engine::scheduler; +namespace text = utils::text; + +using utils::none; +using utils::optional; + + +/// Timeout for the test case cleanup operation. +/// +/// TODO(jmmv): This is here only for testing purposes. Maybe we should expose +/// this setting as part of the user_config. +datetime::delta scheduler::cleanup_timeout(60, 0); + + +/// Timeout for the test case listing operation. +/// +/// TODO(jmmv): This is here only for testing purposes. Maybe we should expose +/// this setting as part of the user_config. +datetime::delta scheduler::list_timeout(300, 0); + + +namespace { + + +/// Magic exit status to indicate that the test case was probably skipped. +/// +/// The test case was only skipped if and only if we return this exit code and +/// we find the skipped_cookie file on disk. +static const int exit_skipped = 84; + + +/// Text file containing the skip reason for the test case. +/// +/// This will only be present within unique_work_directory if the test case +/// exited with the exit_skipped code. However, there is no guarantee that the +/// file is there (say if the test really decided to exit with code exit_skipped +/// on its own). +static const char* skipped_cookie = "skipped.txt"; + + +/// Mapping of interface names to interface definitions. +typedef std::map< std::string, std::shared_ptr< scheduler::interface > > + interfaces_map; + + +/// Mapping of interface names to interface definitions. +/// +/// Use register_interface() to add an entry to this global table. +static interfaces_map interfaces; + + +/// Scans the contents of a directory and appends the file listing to a file. +/// +/// \param dir_path The directory to scan. +/// \param output_file The file to which to append the listing. +/// +/// \throw engine::error If there are problems listing the files. +static void +append_files_listing(const fs::path& dir_path, const fs::path& output_file) +{ + std::ofstream output(output_file.c_str(), std::ios::app); + if (!output) + throw engine::error(F("Failed to open output file %s for append") + % output_file); + try { + std::set < std::string > names; + + const fs::directory dir(dir_path); + for (fs::directory::const_iterator iter = dir.begin(); + iter != dir.end(); ++iter) { + if (iter->name != "." && iter->name != "..") + names.insert(iter->name); + } + + if (!names.empty()) { + output << "Files left in work directory after failure: " + << text::join(names, ", ") << '\n'; + } + } catch (const fs::error& e) { + throw engine::error(F("Cannot append files listing to %s: %s") + % output_file % e.what()); + } +} + + +/// Maintenance data held while a test is being executed. +/// +/// This data structure exists from the moment when a test is executed via +/// scheduler::spawn_test() or scheduler::impl::spawn_cleanup() to when it is +/// cleaned up with result_handle::cleanup(). +/// +/// This is a base data type intended to be extended for the test and cleanup +/// cases so that each contains only the relevant data. +struct exec_data : utils::noncopyable { + /// Test program data for this test case. + const model::test_program_ptr test_program; + + /// Name of the test case. + const std::string test_case_name; + + /// Constructor. + /// + /// \param test_program_ Test program data for this test case. + /// \param test_case_name_ Name of the test case. + exec_data(const model::test_program_ptr test_program_, + const std::string& test_case_name_) : + test_program(test_program_), test_case_name(test_case_name_) + { + } + + /// Destructor. + virtual ~exec_data(void) + { + } +}; + + +/// Maintenance data held while a test is being executed. +struct test_exec_data : public exec_data { + /// Test program-specific execution interface. + const std::shared_ptr< scheduler::interface > interface; + + /// User configuration passed to the execution of the test. We need this + /// here to recover it later when chaining the execution of a cleanup + /// routine (if any). + const config::tree user_config; + + /// Whether this test case still needs to have its cleanup routine executed. + /// + /// This is set externally when the cleanup routine is actually invoked to + /// denote that no further attempts shall be made at cleaning this up. + bool needs_cleanup; + + /// The exit_handle for this test once it has completed. + /// + /// This is set externally when the test case has finished, as we need this + /// information to invoke the followup cleanup routine in the right context, + /// as indicated by needs_cleanup. + optional< executor::exit_handle > exit_handle; + + /// Constructor. + /// + /// \param test_program_ Test program data for this test case. + /// \param test_case_name_ Name of the test case. + /// \param interface_ Test program-specific execution interface. + /// \param user_config_ User configuration passed to the test. + test_exec_data(const model::test_program_ptr test_program_, + const std::string& test_case_name_, + const std::shared_ptr< scheduler::interface > interface_, + const config::tree& user_config_) : + exec_data(test_program_, test_case_name_), + interface(interface_), user_config(user_config_) + { + const model::test_case& test_case = test_program->find(test_case_name); + needs_cleanup = test_case.get_metadata().has_cleanup(); + } +}; + + +/// Maintenance data held while a test cleanup routine is being executed. +/// +/// Instances of this object are related to a previous test_exec_data, as +/// cleanup routines can only exist once the test has been run. +struct cleanup_exec_data : public exec_data { + /// The exit handle of the test. This is necessary so that we can return + /// the correct exit_handle to the user of the scheduler. + executor::exit_handle body_exit_handle; + + /// The final result of the test's body. This is necessary to compute the + /// right return value for a test with a cleanup routine: the body result is + /// respected if it is a "bad" result; else the result of the cleanup + /// routine is used if it has failed. + model::test_result body_result; + + /// Constructor. + /// + /// \param test_program_ Test program data for this test case. + /// \param test_case_name_ Name of the test case. + /// \param body_exit_handle_ If not none, exit handle of the body + /// corresponding to the cleanup routine represented by this exec_data. + /// \param body_result_ If not none, result of the body corresponding to the + /// cleanup routine represented by this exec_data. + cleanup_exec_data(const model::test_program_ptr test_program_, + const std::string& test_case_name_, + const executor::exit_handle& body_exit_handle_, + const model::test_result& body_result_) : + exec_data(test_program_, test_case_name_), + body_exit_handle(body_exit_handle_), body_result(body_result_) + { + } +}; + + +/// Shared pointer to exec_data. +/// +/// We require this because we want exec_data to not be copyable, and thus we +/// cannot just store it in the map without move constructors. +typedef std::shared_ptr< exec_data > exec_data_ptr; + + +/// Mapping of active PIDs to their maintenance data. +typedef std::map< int, exec_data_ptr > exec_data_map; + + +/// Enforces a test program to hold an absolute path. +/// +/// TODO(jmmv): This function (which is a pretty ugly hack) exists because we +/// want the interface hooks to receive a test_program as their argument. +/// However, those hooks run after the test program has been isolated, which +/// means that the current directory has changed since when the test_program +/// objects were created. This causes the absolute_path() method of +/// test_program to return bogus values if the internal representation of their +/// path is relative. We should fix somehow: maybe making the fs module grab +/// its "current_path" view at program startup time; or maybe by grabbing the +/// current path at test_program creation time; or maybe something else. +/// +/// \param program The test program to modify. +/// +/// \return A new test program whose internal paths are absolute. +static model::test_program +force_absolute_paths(const model::test_program program) +{ + const std::string& relative = program.relative_path().str(); + const std::string absolute = program.absolute_path().str(); + + const std::string root = absolute.substr( + 0, absolute.length() - relative.length()); + + return model::test_program( + program.interface_name(), + program.relative_path(), fs::path(root), + program.test_suite_name(), + program.get_metadata(), program.test_cases()); +} + + +/// Functor to list the test cases of a test program. +class list_test_cases { + /// Interface of the test program to execute. + std::shared_ptr< scheduler::interface > _interface; + + /// Test program to execute. + const model::test_program _test_program; + + /// User-provided configuration variables. + const config::tree& _user_config; + +public: + /// Constructor. + /// + /// \param interface Interface of the test program to execute. + /// \param test_program Test program to execute. + /// \param user_config User-provided configuration variables. + list_test_cases( + const std::shared_ptr< scheduler::interface > interface, + const model::test_program* test_program, + const config::tree& user_config) : + _interface(interface), + _test_program(force_absolute_paths(*test_program)), + _user_config(user_config) + { + } + + /// Body of the subprocess. + void + operator()(const fs::path& /* control_directory */) + { + const config::properties_map vars = scheduler::generate_config( + _user_config, _test_program.test_suite_name()); + _interface->exec_list(_test_program, vars); + } +}; + + +/// Functor to execute a test program in a child process. +class run_test_program { + /// Interface of the test program to execute. + std::shared_ptr< scheduler::interface > _interface; + + /// Test program to execute. + const model::test_program _test_program; + + /// Name of the test case to execute. + const std::string& _test_case_name; + + /// User-provided configuration variables. + const config::tree& _user_config; + + /// Verifies if the test case needs to be skipped or not. + /// + /// We could very well run this on the scheduler parent process before + /// issuing the fork. However, doing this here in the child process is + /// better for two reasons: first, it allows us to continue using the simple + /// spawn/wait abstraction of the scheduler; and, second, we parallelize the + /// requirements checks among tests. + /// + /// \post If the test's preconditions are not met, the caller process is + /// terminated with a special exit code and a "skipped cookie" is written to + /// the disk with the reason for the failure. + /// + /// \param skipped_cookie_path File to create with the skip reason details + /// if this test is skipped. + void + do_requirements_check(const fs::path& skipped_cookie_path) + { + const model::test_case& test_case = _test_program.find( + _test_case_name); + + const std::string skip_reason = engine::check_reqs( + test_case.get_metadata(), _user_config, + _test_program.test_suite_name(), + fs::current_path()); + if (skip_reason.empty()) + return; + + std::ofstream output(skipped_cookie_path.c_str()); + if (!output) { + std::perror((F("Failed to open %s for write") % + skipped_cookie_path).str().c_str()); + std::abort(); + } + output << skip_reason; + output.close(); + + // Abruptly terminate the process. We don't want to run any destructors + // inherited from the parent process by mistake, which could, for + // example, delete our own control files! + ::_exit(exit_skipped); + } + +public: + /// Constructor. + /// + /// \param interface Interface of the test program to execute. + /// \param test_program Test program to execute. + /// \param test_case_name Name of the test case to execute. + /// \param user_config User-provided configuration variables. + run_test_program( + const std::shared_ptr< scheduler::interface > interface, + const model::test_program_ptr test_program, + const std::string& test_case_name, + const config::tree& user_config) : + _interface(interface), + _test_program(force_absolute_paths(*test_program)), + _test_case_name(test_case_name), + _user_config(user_config) + { + } + + /// Body of the subprocess. + /// + /// \param control_directory The testcase directory where files will be + /// read from. + void + operator()(const fs::path& control_directory) + { + const model::test_case& test_case = _test_program.find( + _test_case_name); + if (test_case.fake_result()) + ::_exit(EXIT_SUCCESS); + + do_requirements_check(control_directory / skipped_cookie); + + const config::properties_map vars = scheduler::generate_config( + _user_config, _test_program.test_suite_name()); + _interface->exec_test(_test_program, _test_case_name, vars, + control_directory); + } +}; + + +/// Functor to execute a test program in a child process. +class run_test_cleanup { + /// Interface of the test program to execute. + std::shared_ptr< scheduler::interface > _interface; + + /// Test program to execute. + const model::test_program _test_program; + + /// Name of the test case to execute. + const std::string& _test_case_name; + + /// User-provided configuration variables. + const config::tree& _user_config; + +public: + /// Constructor. + /// + /// \param interface Interface of the test program to execute. + /// \param test_program Test program to execute. + /// \param test_case_name Name of the test case to execute. + /// \param user_config User-provided configuration variables. + run_test_cleanup( + const std::shared_ptr< scheduler::interface > interface, + const model::test_program_ptr test_program, + const std::string& test_case_name, + const config::tree& user_config) : + _interface(interface), + _test_program(force_absolute_paths(*test_program)), + _test_case_name(test_case_name), + _user_config(user_config) + { + } + + /// Body of the subprocess. + /// + /// \param control_directory The testcase directory where cleanup will be + /// run from. + void + operator()(const fs::path& control_directory) + { + const config::properties_map vars = scheduler::generate_config( + _user_config, _test_program.test_suite_name()); + _interface->exec_cleanup(_test_program, _test_case_name, vars, + control_directory); + } +}; + + +/// Obtains the right scheduler interface for a given test program. +/// +/// \param name The name of the interface of the test program. +/// +/// \return An scheduler interface. +std::shared_ptr< scheduler::interface > +find_interface(const std::string& name) +{ + const interfaces_map::const_iterator iter = interfaces.find(name); + PRE(interfaces.find(name) != interfaces.end()); + return (*iter).second; +} + + +} // anonymous namespace + + +void +scheduler::interface::exec_cleanup( + const model::test_program& /* test_program */, + const std::string& /* test_case_name */, + const config::properties_map& /* vars */, + const utils::fs::path& /* control_directory */) const +{ + // Most test interfaces do not support standalone cleanup routines so + // provide a default implementation that does nothing. + UNREACHABLE_MSG("exec_cleanup not implemented for an interface that " + "supports standalone cleanup routines"); +} + + +/// Internal implementation of a lazy_test_program. +struct engine::scheduler::lazy_test_program::impl : utils::noncopyable { + /// Whether the test cases list has been yet loaded or not. + bool _loaded; + + /// User configuration to pass to the test program list operation. + config::tree _user_config; + + /// Scheduler context to use to load test cases. + scheduler::scheduler_handle& _scheduler_handle; + + /// Constructor. + /// + /// \param user_config_ User configuration to pass to the test program list + /// operation. + /// \param scheduler_handle_ Scheduler context to use when loading test + /// cases. + impl(const config::tree& user_config_, + scheduler::scheduler_handle& scheduler_handle_) : + _loaded(false), _user_config(user_config_), + _scheduler_handle(scheduler_handle_) + { + } +}; + + +/// Constructs a new test program. +/// +/// \param interface_name_ Name of the test program interface. +/// \param binary_ The name of the test program binary relative to root_. +/// \param root_ The root of the test suite containing the test program. +/// \param test_suite_name_ The name of the test suite this program belongs to. +/// \param md_ Metadata of the test program. +/// \param user_config_ User configuration to pass to the scheduler. +/// \param scheduler_handle_ Scheduler context to use to load test cases. +scheduler::lazy_test_program::lazy_test_program( + const std::string& interface_name_, + const fs::path& binary_, + const fs::path& root_, + const std::string& test_suite_name_, + const model::metadata& md_, + const config::tree& user_config_, + scheduler::scheduler_handle& scheduler_handle_) : + test_program(interface_name_, binary_, root_, test_suite_name_, md_, + model::test_cases_map()), + _pimpl(new impl(user_config_, scheduler_handle_)) +{ +} + + +/// Gets or loads the list of test cases from the test program. +/// +/// \return The list of test cases provided by the test program. +const model::test_cases_map& +scheduler::lazy_test_program::test_cases(void) const +{ + _pimpl->_scheduler_handle.check_interrupt(); + + if (!_pimpl->_loaded) { + const model::test_cases_map tcs = _pimpl->_scheduler_handle.list_tests( + this, _pimpl->_user_config); + + // Due to the restrictions on when set_test_cases() may be called (as a + // way to lazily initialize the test cases list before it is ever + // returned), this cast is valid. + const_cast< scheduler::lazy_test_program* >(this)->set_test_cases(tcs); + + _pimpl->_loaded = true; + + _pimpl->_scheduler_handle.check_interrupt(); + } + + INV(_pimpl->_loaded); + return test_program::test_cases(); +} + + +/// Internal implementation for the result_handle class. +struct engine::scheduler::result_handle::bimpl : utils::noncopyable { + /// Generic executor exit handle for this result handle. + executor::exit_handle generic; + + /// Mutable pointer to the corresponding scheduler state. + /// + /// This object references a member of the scheduler_handle that yielded + /// this result_handle instance. We need this direct access to clean up + /// after ourselves when the result is destroyed. + exec_data_map& all_exec_data; + + /// Constructor. + /// + /// \param generic_ Generic executor exit handle for this result handle. + /// \param [in,out] all_exec_data_ Global object keeping track of all active + /// executions for an scheduler. This is a pointer to a member of the + /// scheduler_handle object. + bimpl(const executor::exit_handle generic_, exec_data_map& all_exec_data_) : + generic(generic_), all_exec_data(all_exec_data_) + { + } + + /// Destructor. + ~bimpl(void) + { + LD(F("Removing %s from all_exec_data") % generic.original_pid()); + all_exec_data.erase(generic.original_pid()); + } +}; + + +/// Constructor. +/// +/// \param pbimpl Constructed internal implementation. +scheduler::result_handle::result_handle(std::shared_ptr< bimpl > pbimpl) : + _pbimpl(pbimpl) +{ +} + + +/// Destructor. +scheduler::result_handle::~result_handle(void) +{ +} + + +/// Cleans up the test case results. +/// +/// This function should be called explicitly as it provides the means to +/// control any exceptions raised during cleanup. Do not rely on the destructor +/// to clean things up. +/// +/// \throw engine::error If the cleanup fails, especially due to the inability +/// to remove the work directory. +void +scheduler::result_handle::cleanup(void) +{ + _pbimpl->generic.cleanup(); +} + + +/// Returns the original PID corresponding to this result. +/// +/// \return An exec_handle. +int +scheduler::result_handle::original_pid(void) const +{ + return _pbimpl->generic.original_pid(); +} + + +/// Returns the timestamp of when spawn_test was called. +/// +/// \return A timestamp. +const datetime::timestamp& +scheduler::result_handle::start_time(void) const +{ + return _pbimpl->generic.start_time(); +} + + +/// Returns the timestamp of when wait_any_test returned this object. +/// +/// \return A timestamp. +const datetime::timestamp& +scheduler::result_handle::end_time(void) const +{ + return _pbimpl->generic.end_time(); +} + + +/// Returns the path to the test-specific work directory. +/// +/// This is guaranteed to be clear of files created by the scheduler. +/// +/// \return The path to a directory that exists until cleanup() is called. +fs::path +scheduler::result_handle::work_directory(void) const +{ + return _pbimpl->generic.work_directory(); +} + + +/// Returns the path to the test's stdout file. +/// +/// \return The path to a file that exists until cleanup() is called. +const fs::path& +scheduler::result_handle::stdout_file(void) const +{ + return _pbimpl->generic.stdout_file(); +} + + +/// Returns the path to the test's stderr file. +/// +/// \return The path to a file that exists until cleanup() is called. +const fs::path& +scheduler::result_handle::stderr_file(void) const +{ + return _pbimpl->generic.stderr_file(); +} + + +/// Internal implementation for the test_result_handle class. +struct engine::scheduler::test_result_handle::impl : utils::noncopyable { + /// Test program data for this test case. + model::test_program_ptr test_program; + + /// Name of the test case. + std::string test_case_name; + + /// The actual result of the test execution. + const model::test_result test_result; + + /// Constructor. + /// + /// \param test_program_ Test program data for this test case. + /// \param test_case_name_ Name of the test case. + /// \param test_result_ The actual result of the test execution. + impl(const model::test_program_ptr test_program_, + const std::string& test_case_name_, + const model::test_result& test_result_) : + test_program(test_program_), + test_case_name(test_case_name_), + test_result(test_result_) + { + } +}; + + +/// Constructor. +/// +/// \param pbimpl Constructed internal implementation for the base object. +/// \param pimpl Constructed internal implementation. +scheduler::test_result_handle::test_result_handle( + std::shared_ptr< bimpl > pbimpl, std::shared_ptr< impl > pimpl) : + result_handle(pbimpl), _pimpl(pimpl) +{ +} + + +/// Destructor. +scheduler::test_result_handle::~test_result_handle(void) +{ +} + + +/// Returns the test program that yielded this result. +/// +/// \return A test program. +const model::test_program_ptr +scheduler::test_result_handle::test_program(void) const +{ + return _pimpl->test_program; +} + + +/// Returns the name of the test case that yielded this result. +/// +/// \return A test case name +const std::string& +scheduler::test_result_handle::test_case_name(void) const +{ + return _pimpl->test_case_name; +} + + +/// Returns the actual result of the test execution. +/// +/// \return A test result. +const model::test_result& +scheduler::test_result_handle::test_result(void) const +{ + return _pimpl->test_result; +} + + +/// Internal implementation for the scheduler_handle. +struct engine::scheduler::scheduler_handle::impl : utils::noncopyable { + /// Generic executor instance encapsulated by this one. + executor::executor_handle generic; + + /// Mapping of exec handles to the data required at run time. + exec_data_map all_exec_data; + + /// Collection of test_exec_data objects. + typedef std::vector< const test_exec_data* > test_exec_data_vector; + + /// Constructor. + impl(void) : generic(executor::setup()) + { + } + + /// Destructor. + /// + /// This runs any pending cleanup routines, which should only happen if the + /// scheduler is abruptly terminated (aka if a signal is received). + ~impl(void) + { + const test_exec_data_vector tests_data = tests_needing_cleanup(); + + for (test_exec_data_vector::const_iterator iter = tests_data.begin(); + iter != tests_data.end(); ++iter) { + const test_exec_data* test_data = *iter; + + try { + sync_cleanup(test_data); + } catch (const std::runtime_error& e) { + LW(F("Failed to run cleanup routine for %s:%s on abrupt " + "termination") + % test_data->test_program->relative_path() + % test_data->test_case_name); + } + } + } + + /// Finds any pending exec_datas that correspond to tests needing cleanup. + /// + /// \return The collection of test_exec_data objects that have their + /// needs_cleanup property set to true. + test_exec_data_vector + tests_needing_cleanup(void) + { + test_exec_data_vector tests_data; + + for (exec_data_map::const_iterator iter = all_exec_data.begin(); + iter != all_exec_data.end(); ++iter) { + const exec_data_ptr data = (*iter).second; + + try { + test_exec_data* test_data = &dynamic_cast< test_exec_data& >( + *data.get()); + if (test_data->needs_cleanup) { + tests_data.push_back(test_data); + test_data->needs_cleanup = false; + } + } catch (const std::bad_cast& e) { + // Do nothing for cleanup_exec_data objects. + } + } + + return tests_data; + } + + /// Cleans up a single test case synchronously. + /// + /// \param test_data The data of the previously executed test case to be + /// cleaned up. + void + sync_cleanup(const test_exec_data* test_data) + { + // The message in this result should never be seen by the user, but use + // something reasonable just in case it leaks and we need to pinpoint + // the call site. + model::test_result result(model::test_result_broken, + "Test case died abruptly"); + + const executor::exec_handle cleanup_handle = spawn_cleanup( + test_data->test_program, test_data->test_case_name, + test_data->user_config, test_data->exit_handle.get(), + result); + generic.wait(cleanup_handle); + } + + /// Forks and executes a test case cleanup routine asynchronously. + /// + /// \param test_program The container test program. + /// \param test_case_name The name of the test case to run. + /// \param user_config User-provided configuration variables. + /// \param body_handle The exit handle of the test case's corresponding + /// body. The cleanup will be executed in the same context. + /// \param body_result The result of the test case's corresponding body. + /// + /// \return A handle for the background operation. Used to match the result + /// of the execution returned by wait_any() with this invocation. + executor::exec_handle + spawn_cleanup(const model::test_program_ptr test_program, + const std::string& test_case_name, + const config::tree& user_config, + const executor::exit_handle& body_handle, + const model::test_result& body_result) + { + generic.check_interrupt(); + + const std::shared_ptr< scheduler::interface > interface = + find_interface(test_program->interface_name()); + + LI(F("Spawning %s:%s (cleanup)") % test_program->absolute_path() % + test_case_name); + + const executor::exec_handle handle = generic.spawn_followup( + run_test_cleanup(interface, test_program, test_case_name, + user_config), + body_handle, cleanup_timeout); + + const exec_data_ptr data(new cleanup_exec_data( + test_program, test_case_name, body_handle, body_result)); + LD(F("Inserting %s into all_exec_data (cleanup)") % handle.pid()); + INV_MSG(all_exec_data.find(handle.pid()) == all_exec_data.end(), + F("PID %s already in all_exec_data; not properly cleaned " + "up or reused too fast") % handle.pid());; + all_exec_data.insert(exec_data_map::value_type(handle.pid(), data)); + + return handle; + } +}; + + +/// Constructor. +scheduler::scheduler_handle::scheduler_handle(void) : _pimpl(new impl()) +{ +} + + +/// Destructor. +scheduler::scheduler_handle::~scheduler_handle(void) +{ +} + + +/// Queries the path to the root of the work directory for all tests. +/// +/// \return A path. +const fs::path& +scheduler::scheduler_handle::root_work_directory(void) const +{ + return _pimpl->generic.root_work_directory(); +} + + +/// Cleans up the scheduler state. +/// +/// This function should be called explicitly as it provides the means to +/// control any exceptions raised during cleanup. Do not rely on the destructor +/// to clean things up. +/// +/// \throw engine::error If there are problems cleaning up the scheduler. +void +scheduler::scheduler_handle::cleanup(void) +{ + _pimpl->generic.cleanup(); +} + + +/// Checks if the given interface name is valid. +/// +/// \param name The name of the interface to validate. +/// +/// \throw engine::error If the given interface is not supported. +void +scheduler::ensure_valid_interface(const std::string& name) +{ + if (interfaces.find(name) == interfaces.end()) + throw engine::error(F("Unsupported test interface '%s'") % name); +} + + +/// Registers a new interface. +/// +/// \param name The name of the interface. Must not have yet been registered. +/// \param spec Interface specification. +void +scheduler::register_interface(const std::string& name, + const std::shared_ptr< interface > spec) +{ + PRE(interfaces.find(name) == interfaces.end()); + interfaces.insert(interfaces_map::value_type(name, spec)); +} + + +/// Returns the names of all registered interfaces. +/// +/// \return A collection of interface names. +std::set< std::string > +scheduler::registered_interface_names(void) +{ + std::set< std::string > names; + for (interfaces_map::const_iterator iter = interfaces.begin(); + iter != interfaces.end(); ++iter) { + names.insert((*iter).first); + } + return names; +} + + +/// Initializes the scheduler. +/// +/// \pre This function can only be called if there is no other scheduler_handle +/// object alive. +/// +/// \return A handle to the operations of the scheduler. +scheduler::scheduler_handle +scheduler::setup(void) +{ + return scheduler_handle(); +} + + +/// Retrieves the list of test cases from a test program. +/// +/// This operation is currently synchronous. +/// +/// This operation should never throw. Any errors during the processing of the +/// test case list are subsumed into a single test case in the return value that +/// represents the failed retrieval. +/// +/// \param test_program The test program from which to obtain the list of test +/// cases. +/// \param user_config User-provided configuration variables. +/// +/// \return The list of test cases. +model::test_cases_map +scheduler::scheduler_handle::list_tests( + const model::test_program* test_program, + const config::tree& user_config) +{ + _pimpl->generic.check_interrupt(); + + const std::shared_ptr< scheduler::interface > interface = find_interface( + test_program->interface_name()); + + try { + const executor::exec_handle exec_handle = _pimpl->generic.spawn( + list_test_cases(interface, test_program, user_config), + list_timeout, none); + executor::exit_handle exit_handle = _pimpl->generic.wait(exec_handle); + + const model::test_cases_map test_cases = interface->parse_list( + exit_handle.status(), + exit_handle.stdout_file(), + exit_handle.stderr_file()); + + exit_handle.cleanup(); + + if (test_cases.empty()) + throw std::runtime_error("Empty test cases list"); + + return test_cases; + } catch (const std::runtime_error& e) { + // TODO(jmmv): This is a very ugly workaround for the fact that we + // cannot report failures at the test-program level. + LW(F("Failed to load test cases list: %s") % e.what()); + model::test_cases_map fake_test_cases; + fake_test_cases.insert(model::test_cases_map::value_type( + "__test_cases_list__", + model::test_case( + "__test_cases_list__", + "Represents the correct processing of the test cases list", + model::test_result(model::test_result_broken, e.what())))); + return fake_test_cases; + } +} + + +/// Forks and executes a test case asynchronously. +/// +/// Note that the caller needn't know if the test has a cleanup routine or not. +/// If there indeed is a cleanup routine, we trigger it at wait_any() time. +/// +/// \param test_program The container test program. +/// \param test_case_name The name of the test case to run. +/// \param user_config User-provided configuration variables. +/// +/// \return A handle for the background operation. Used to match the result of +/// the execution returned by wait_any() with this invocation. +scheduler::exec_handle +scheduler::scheduler_handle::spawn_test( + const model::test_program_ptr test_program, + const std::string& test_case_name, + const config::tree& user_config) +{ + _pimpl->generic.check_interrupt(); + + const std::shared_ptr< scheduler::interface > interface = find_interface( + test_program->interface_name()); + + LI(F("Spawning %s:%s") % test_program->absolute_path() % test_case_name); + + const model::test_case& test_case = test_program->find(test_case_name); + + optional< passwd::user > unprivileged_user; + if (user_config.is_set("unprivileged_user") && + test_case.get_metadata().required_user() == "unprivileged") { + unprivileged_user = user_config.lookup< engine::user_node >( + "unprivileged_user"); + } + + const executor::exec_handle handle = _pimpl->generic.spawn( + run_test_program(interface, test_program, test_case_name, + user_config), + test_case.get_metadata().timeout(), + unprivileged_user); + + const exec_data_ptr data(new test_exec_data( + test_program, test_case_name, interface, user_config)); + LD(F("Inserting %s into all_exec_data") % handle.pid()); + INV_MSG( + _pimpl->all_exec_data.find(handle.pid()) == _pimpl->all_exec_data.end(), + F("PID %s already in all_exec_data; not cleaned up or reused too fast") + % handle.pid());; + _pimpl->all_exec_data.insert(exec_data_map::value_type(handle.pid(), data)); + + return handle.pid(); +} + + +/// Waits for completion of any forked test case. +/// +/// Note that if the terminated test case has a cleanup routine, this function +/// is the one in charge of spawning the cleanup routine asynchronously. +/// +/// \return The result of the execution of a subprocess. This is a dynamically +/// allocated object because the scheduler can spawn subprocesses of various +/// types and, at wait time, we don't know upfront what we are going to get. +scheduler::result_handle_ptr +scheduler::scheduler_handle::wait_any(void) +{ + _pimpl->generic.check_interrupt(); + + executor::exit_handle handle = _pimpl->generic.wait_any(); + + const exec_data_map::iterator iter = _pimpl->all_exec_data.find( + handle.original_pid()); + exec_data_ptr data = (*iter).second; + + utils::dump_stacktrace_if_available(data->test_program->absolute_path(), + _pimpl->generic, handle); + + optional< model::test_result > result; + try { + test_exec_data* test_data = &dynamic_cast< test_exec_data& >( + *data.get()); + LD(F("Got %s from all_exec_data") % handle.original_pid()); + + test_data->exit_handle = handle; + + const model::test_case& test_case = test_data->test_program->find( + test_data->test_case_name); + + result = test_case.fake_result(); + + if (!result && handle.status() && handle.status().get().exited() && + handle.status().get().exitstatus() == exit_skipped) { + // If the test's process terminated with our magic "exit_skipped" + // status, there are two cases to handle. The first is the case + // where the "skipped cookie" exists, in which case we never got to + // actually invoke the test program; if that's the case, handle it + // here. The second case is where the test case actually decided to + // exit with the "exit_skipped" status; in that case, just fall back + // to the regular status handling. + const fs::path skipped_cookie_path = handle.control_directory() / + skipped_cookie; + std::ifstream input(skipped_cookie_path.c_str()); + if (input) { + result = model::test_result(model::test_result_skipped, + utils::read_stream(input)); + input.close(); + + // If we determined that the test needs to be skipped, we do not + // want to run the cleanup routine because doing so could result + // in errors. However, we still want to run the cleanup routine + // if the test's body reports a skip (because actions could have + // already been taken). + test_data->needs_cleanup = false; + } + } + if (!result) { + result = test_data->interface->compute_result( + handle.status(), + handle.control_directory(), + handle.stdout_file(), + handle.stderr_file()); + } + INV(result); + + if (!result.get().good()) { + append_files_listing(handle.work_directory(), + handle.stderr_file()); + } + + if (test_data->needs_cleanup) { + INV(test_case.get_metadata().has_cleanup()); + // The test body has completed and we have processed it. If there + // is a cleanup routine, trigger it now and wait for any other test + // completion. The caller never knows about cleanup routines. + _pimpl->spawn_cleanup(test_data->test_program, + test_data->test_case_name, + test_data->user_config, handle, result.get()); + test_data->needs_cleanup = false; + + // TODO(jmmv): Chaining this call is ugly. We'd be better off by + // looping over terminated processes until we got a result suitable + // for user consumption. For the time being this is good enough and + // not a problem because the call chain won't get big: the majority + // of test cases do not have cleanup routines. + return wait_any(); + } + } catch (const std::bad_cast& e) { + const cleanup_exec_data* cleanup_data = + &dynamic_cast< const cleanup_exec_data& >(*data.get()); + LD(F("Got %s from all_exec_data (cleanup)") % handle.original_pid()); + + // Handle the completion of cleanup subprocesses internally: the caller + // is not aware that these exist so, when we return, we must return the + // data for the original test that triggered this routine. For example, + // because the caller wants to see the exact same exec_handle that was + // returned by spawn_test. + + const model::test_result& body_result = cleanup_data->body_result; + if (body_result.good()) { + if (!handle.status()) { + result = model::test_result(model::test_result_broken, + "Test case cleanup timed out"); + } else { + if (!handle.status().get().exited() || + handle.status().get().exitstatus() != EXIT_SUCCESS) { + result = model::test_result( + model::test_result_broken, + "Test case cleanup did not terminate successfully"); + } else { + result = body_result; + } + } + } else { + result = body_result; + } + + // Untrack the cleanup process. This must be done explicitly because we + // do not create a result_handle object for the cleanup, and that is the + // one in charge of doing so in the regular (non-cleanup) case. + LD(F("Removing %s from all_exec_data (cleanup) in favor of %s") + % handle.original_pid() + % cleanup_data->body_exit_handle.original_pid()); + _pimpl->all_exec_data.erase(handle.original_pid()); + + handle = cleanup_data->body_exit_handle; + } + INV(result); + + std::shared_ptr< result_handle::bimpl > result_handle_bimpl( + new result_handle::bimpl(handle, _pimpl->all_exec_data)); + std::shared_ptr< test_result_handle::impl > test_result_handle_impl( + new test_result_handle::impl( + data->test_program, data->test_case_name, result.get())); + return result_handle_ptr(new test_result_handle(result_handle_bimpl, + test_result_handle_impl)); +} + + +/// Forks and executes a test case synchronously for debugging. +/// +/// \pre No other processes should be in execution by the scheduler. +/// +/// \param test_program The container test program. +/// \param test_case_name The name of the test case to run. +/// \param user_config User-provided configuration variables. +/// \param stdout_target File to which to write the stdout of the test case. +/// \param stderr_target File to which to write the stderr of the test case. +/// +/// \return The result of the execution of the test. +scheduler::result_handle_ptr +scheduler::scheduler_handle::debug_test( + const model::test_program_ptr test_program, + const std::string& test_case_name, + const config::tree& user_config, + const fs::path& stdout_target, + const fs::path& stderr_target) +{ + const exec_handle exec_handle = spawn_test( + test_program, test_case_name, user_config); + result_handle_ptr result_handle = wait_any(); + + // TODO(jmmv): We need to do this while the subprocess is alive. This is + // important for debugging purposes, as we should see the contents of stdout + // or stderr as they come in. + // + // Unfortunately, we cannot do so. We cannot just read and block from a + // file, waiting for further output to appear... as this only works on pipes + // or sockets. We need a better interface for this whole thing. + { + std::auto_ptr< std::ostream > output = utils::open_ostream( + stdout_target); + *output << utils::read_file(result_handle->stdout_file()); + } + { + std::auto_ptr< std::ostream > output = utils::open_ostream( + stderr_target); + *output << utils::read_file(result_handle->stderr_file()); + } + + INV(result_handle->original_pid() == exec_handle); + return result_handle; +} + + +/// Checks if an interrupt has fired. +/// +/// Calls to this function should be sprinkled in strategic places through the +/// code protected by an interrupts_handler object. +/// +/// This is just a wrapper over signals::check_interrupt() to avoid leaking this +/// dependency to the caller. +/// +/// \throw signals::interrupted_error If there has been an interrupt. +void +scheduler::scheduler_handle::check_interrupt(void) const +{ + _pimpl->generic.check_interrupt(); +} + + +/// Queries the current execution context. +/// +/// \return The queried context. +model::context +scheduler::current_context(void) +{ + return model::context(fs::current_path(), utils::getallenv()); +} + + +/// Generates the set of configuration variables for a test program. +/// +/// \param user_config The configuration variables provided by the user. +/// \param test_suite The name of the test suite. +/// +/// \return The mapping of configuration variables for the test program. +config::properties_map +scheduler::generate_config(const config::tree& user_config, + const std::string& test_suite) +{ + config::properties_map props; + + try { + props = user_config.all_properties(F("test_suites.%s") % test_suite, + true); + } catch (const config::unknown_key_error& unused_error) { + // Ignore: not all test suites have entries in the configuration. + } + + // TODO(jmmv): This is a hack that exists for the ATF interface only, so it + // should be moved there. + if (user_config.is_set("unprivileged_user")) { + const passwd::user& user = + user_config.lookup< engine::user_node >("unprivileged_user"); + props["unprivileged-user"] = user.name; + } + + return props; +} diff --git a/engine/scheduler.hpp b/engine/scheduler.hpp new file mode 100644 index 000000000000..24ff0b5a26fc --- /dev/null +++ b/engine/scheduler.hpp @@ -0,0 +1,282 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/scheduler.hpp +/// Multiprogrammed executor of test related operations. +/// +/// The scheduler's public interface exposes test cases as "black boxes". The +/// handling of cleanup routines is completely hidden from the caller and +/// happens in two cases: first, once a test case completes; and, second, in the +/// case of abrupt termination due to the reception of a signal. +/// +/// Hiding cleanup routines from the caller is an attempt to keep the logic of +/// execution and results handling in a single place. Otherwise, the various +/// drivers (say run_tests and debug_test) would need to replicate the handling +/// of this logic, which is tricky in itself (particularly due to signal +/// handling) and would lead to inconsistencies. +/// +/// Handling cleanup routines in the manner described above is *incredibly +/// complicated* (insane, actually) as you will see from the code. The +/// complexity will bite us in the future (today is 2015-06-26). Switching to a +/// threads-based implementation would probably simplify the code flow +/// significantly and allow parallelization of the test case listings in a +/// reasonable manner, though it depends on whether we can get clean handling of +/// signals and on whether we could use C++11's std::thread. (Is this a to-do? +/// Maybe. Maybe not.) +/// +/// See the documentation in utils/process/executor.hpp for details on +/// the expected workflow of these classes. + +#if !defined(ENGINE_SCHEDULER_HPP) +#define ENGINE_SCHEDULER_HPP + +#include "engine/scheduler_fwd.hpp" + +#include <memory> +#include <set> +#include <string> + +#include "model/context_fwd.hpp" +#include "model/metadata_fwd.hpp" +#include "model/test_case_fwd.hpp" +#include "model/test_program.hpp" +#include "model/test_result_fwd.hpp" +#include "utils/config/tree_fwd.hpp" +#include "utils/datetime_fwd.hpp" +#include "utils/defs.hpp" +#include "utils/fs/path_fwd.hpp" +#include "utils/optional.hpp" +#include "utils/process/executor_fwd.hpp" +#include "utils/process/status_fwd.hpp" + +namespace engine { +namespace scheduler { + + +/// Abstract interface of a test program scheduler interface. +/// +/// This interface defines the test program-specific operations that need to be +/// invoked at different points during the execution of a given test case. The +/// scheduler internally instantiates one of these for every test case. +class interface { +public: + /// Destructor. + virtual ~interface() {} + + /// Executes a test program's list operation. + /// + /// This method is intended to be called within a subprocess and is expected + /// to terminate execution either by exec(2)ing the test program or by + /// exiting with a failure. + /// + /// \param test_program The test program to execute. + /// \param vars User-provided variables to pass to the test program. + virtual void exec_list(const model::test_program& test_program, + const utils::config::properties_map& vars) + const UTILS_NORETURN = 0; + + /// Computes the test cases list of a test program. + /// + /// \param status The termination status of the subprocess used to execute + /// the exec_test() method or none if the test timed out. + /// \param stdout_path Path to the file containing the stdout of the test. + /// \param stderr_path Path to the file containing the stderr of the test. + /// + /// \return A list of test cases. + virtual model::test_cases_map parse_list( + const utils::optional< utils::process::status >& status, + const utils::fs::path& stdout_path, + const utils::fs::path& stderr_path) const = 0; + + /// Executes a test case of the test program. + /// + /// This method is intended to be called within a subprocess and is expected + /// to terminate execution either by exec(2)ing the test program or by + /// exiting with a failure. + /// + /// \param test_program The test program to execute. + /// \param test_case_name Name of the test case to invoke. + /// \param vars User-provided variables to pass to the test program. + /// \param control_directory Directory where the interface may place control + /// files. + virtual void exec_test(const model::test_program& test_program, + const std::string& test_case_name, + const utils::config::properties_map& vars, + const utils::fs::path& control_directory) + const UTILS_NORETURN = 0; + + /// Executes a test cleanup routine of the test program. + /// + /// This method is intended to be called within a subprocess and is expected + /// to terminate execution either by exec(2)ing the test program or by + /// exiting with a failure. + /// + /// \param test_program The test program to execute. + /// \param test_case_name Name of the test case to invoke. + /// \param vars User-provided variables to pass to the test program. + /// \param control_directory Directory where the interface may place control + /// files. + virtual void exec_cleanup(const model::test_program& test_program, + const std::string& test_case_name, + const utils::config::properties_map& vars, + const utils::fs::path& control_directory) + const UTILS_NORETURN; + + /// Computes the result of a test case based on its termination status. + /// + /// \param status The termination status of the subprocess used to execute + /// the exec_test() method or none if the test timed out. + /// \param control_directory Directory where the interface may have placed + /// control files. + /// \param stdout_path Path to the file containing the stdout of the test. + /// \param stderr_path Path to the file containing the stderr of the test. + /// + /// \return A test result. + virtual model::test_result compute_result( + const utils::optional< utils::process::status >& status, + const utils::fs::path& control_directory, + const utils::fs::path& stdout_path, + const utils::fs::path& stderr_path) const = 0; +}; + + +/// Implementation of a test program with lazy loading of test cases. +class lazy_test_program : public model::test_program { + struct impl; + + /// Pointer to the shared internal implementation. + std::shared_ptr< impl > _pimpl; + +public: + lazy_test_program(const std::string&, const utils::fs::path&, + const utils::fs::path&, const std::string&, + const model::metadata&, + const utils::config::tree&, + scheduler_handle&); + + const model::test_cases_map& test_cases(void) const; +}; + + +/// Base type containing the results of the execution of a subprocess. +class result_handle { +protected: + struct bimpl; + +private: + /// Pointer to internal implementation of the base type. + std::shared_ptr< bimpl > _pbimpl; + +protected: + friend class scheduler_handle; + result_handle(std::shared_ptr< bimpl >); + +public: + virtual ~result_handle(void) = 0; + + void cleanup(void); + + int original_pid(void) const; + const utils::datetime::timestamp& start_time() const; + const utils::datetime::timestamp& end_time() const; + utils::fs::path work_directory(void) const; + const utils::fs::path& stdout_file(void) const; + const utils::fs::path& stderr_file(void) const; +}; + + +/// Container for all test termination data and accessor to cleanup operations. +class test_result_handle : public result_handle { + struct impl; + /// Pointer to internal implementation. + std::shared_ptr< impl > _pimpl; + + friend class scheduler_handle; + test_result_handle(std::shared_ptr< bimpl >, std::shared_ptr< impl >); + +public: + ~test_result_handle(void); + + const model::test_program_ptr test_program(void) const; + const std::string& test_case_name(void) const; + const model::test_result& test_result(void) const; +}; + + +/// Stateful interface to the multiprogrammed execution of tests. +class scheduler_handle { + struct impl; + /// Pointer to internal implementation. + std::shared_ptr< impl > _pimpl; + + friend scheduler_handle setup(void); + scheduler_handle(void); + +public: + ~scheduler_handle(void); + + const utils::fs::path& root_work_directory(void) const; + + void cleanup(void); + + model::test_cases_map list_tests(const model::test_program*, + const utils::config::tree&); + exec_handle spawn_test(const model::test_program_ptr, + const std::string&, + const utils::config::tree&); + result_handle_ptr wait_any(void); + + result_handle_ptr debug_test(const model::test_program_ptr, + const std::string&, + const utils::config::tree&, + const utils::fs::path&, + const utils::fs::path&); + + void check_interrupt(void) const; +}; + + +extern utils::datetime::delta cleanup_timeout; +extern utils::datetime::delta list_timeout; + + +void ensure_valid_interface(const std::string&); +void register_interface(const std::string&, const std::shared_ptr< interface >); +std::set< std::string > registered_interface_names(void); +scheduler_handle setup(void); + +model::context current_context(void); +utils::config::properties_map generate_config(const utils::config::tree&, + const std::string&); + + +} // namespace scheduler +} // namespace engine + + +#endif // !defined(ENGINE_SCHEDULER_HPP) diff --git a/engine/scheduler_fwd.hpp b/engine/scheduler_fwd.hpp new file mode 100644 index 000000000000..f61b084e5a8d --- /dev/null +++ b/engine/scheduler_fwd.hpp @@ -0,0 +1,61 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/scheduler_fwd.hpp +/// Forward declarations for engine/scheduler.hpp + +#if !defined(ENGINE_SCHEDULER_FWD_HPP) +#define ENGINE_SCHEDULER_FWD_HPP + +#include <memory> + +namespace engine { +namespace scheduler { + + +/// Unique identifier for in-flight execution operations. +/// +/// TODO(jmmv): Might be worth to drop altogether and just use "int". The type +/// difference with executor::exec_handle is confusing. +typedef int exec_handle; + + +class scheduler_handle; +class interface; +class result_handle; +class test_result_handle; + + +/// Pointer to a dynamically-allocated result_handle. +typedef std::shared_ptr< result_handle > result_handle_ptr; + + +} // namespace scheduler +} // namespace engine + +#endif // !defined(ENGINE_SCHEDULER_FWD_HPP) diff --git a/engine/scheduler_test.cpp b/engine/scheduler_test.cpp new file mode 100644 index 000000000000..e144761d8f01 --- /dev/null +++ b/engine/scheduler_test.cpp @@ -0,0 +1,1242 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/scheduler.hpp" + +extern "C" { +#include <sys/types.h> + +#include <signal.h> +#include <unistd.h> +} + +#include <cstdlib> +#include <fstream> +#include <iostream> +#include <string> + +#include <atf-c++.hpp> + +#include "engine/config.hpp" +#include "engine/exceptions.hpp" +#include "model/context.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/defs.hpp" +#include "utils/env.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/passwd.hpp" +#include "utils/process/status.hpp" +#include "utils/sanity.hpp" +#include "utils/stacktrace.hpp" +#include "utils/stream.hpp" +#include "utils/test_utils.ipp" +#include "utils/text/exceptions.hpp" +#include "utils/text/operations.ipp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace passwd = utils::passwd; +namespace process = utils::process; +namespace scheduler = engine::scheduler; +namespace text = utils::text; + +using utils::none; +using utils::optional; + + +namespace { + + +/// Checks if a string starts with a prefix. +/// +/// \param str The string to be tested. +/// \param prefix The prefix to look for. +/// +/// \return True if the string is prefixed as specified. +static bool +starts_with(const std::string& str, const std::string& prefix) +{ + return (str.length() >= prefix.length() && + str.substr(0, prefix.length()) == prefix); +} + + +/// Strips a prefix from a string and converts the rest to an integer. +/// +/// \param str The string to be tested. +/// \param prefix The prefix to strip from the string. +/// +/// \return The part of the string after the prefix converted to an integer. +static int +suffix_to_int(const std::string& str, const std::string& prefix) +{ + PRE(starts_with(str, prefix)); + try { + return text::to_type< int >(str.substr(prefix.length())); + } catch (const text::value_error& error) { + std::cerr << F("Failed: %s\n") % error.what(); + std::abort(); + } +} + + +/// Mock interface definition for testing. +/// +/// This scheduler interface does not execute external binaries. It is designed +/// to simulate the scheduler of various programs with different exit statuses. +class mock_interface : public scheduler::interface { + /// Executes the subprocess simulating an exec. + /// + /// This is just a simple wrapper over _exit(2) because we cannot use + /// std::exit on exit from this mock interface. The reason is that we do + /// not want to invoke any destructors as otherwise we'd clear up the global + /// scheduler state by mistake. This wouldn't be a major problem if it + /// wasn't because doing so deletes on-disk files and we want to leave them + /// in place so that the parent process can test for them! + /// + /// \param exit_code Exit code. + void + do_exit(const int exit_code) const UTILS_NORETURN + { + std::cout.flush(); + std::cerr.flush(); + ::_exit(exit_code); + } + + /// Executes a test case that creates various files and then fails. + void + exec_create_files_and_fail(void) const UTILS_NORETURN + { + std::cerr << "This should not be clobbered\n"; + atf::utils::create_file("first file", ""); + atf::utils::create_file("second-file", ""); + fs::mkdir_p(fs::path("dir1/dir2"), 0755); + ::kill(::getpid(), SIGTERM); + std::abort(); + } + + /// Executes a test case that deletes all files in the current directory. + /// + /// This is intended to validate that the test runs in an empty directory, + /// separate from any control files that the scheduler may have created. + void + exec_delete_all(void) const UTILS_NORETURN + { + const int exit_code = ::system("rm *") == -1 + ? EXIT_FAILURE : EXIT_SUCCESS; + + // Recreate our own cookie. + atf::utils::create_file("exec_test_was_called", ""); + + do_exit(exit_code); + } + + /// Executes a test case that returns a specific exit code. + /// + /// \param exit_code Exit status to terminate the program with. + void + exec_exit(const int exit_code) const UTILS_NORETURN + { + do_exit(exit_code); + } + + /// Executes a test case that just fails. + void + exec_fail(void) const UTILS_NORETURN + { + std::cerr << "This should not be clobbered\n"; + ::kill(::getpid(), SIGTERM); + std::abort(); + } + + /// Executes a test case that prints all input parameters to the functor. + /// + /// \param test_program The test program to execute. + /// \param test_case_name Name of the test case to invoke, which must be a + /// number. + /// \param vars User-provided variables to pass to the test program. + void + exec_print_params(const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars) const + UTILS_NORETURN + { + std::cout << F("Test program: %s\n") % test_program.relative_path(); + std::cout << F("Test case: %s\n") % test_case_name; + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + std::cout << F("%s=%s\n") % (*iter).first % (*iter).second; + } + + std::cerr << F("stderr: %s\n") % test_case_name; + + do_exit(EXIT_SUCCESS); + } + +public: + /// Executes a test program's list operation. + /// + /// This method is intended to be called within a subprocess and is expected + /// to terminate execution either by exec(2)ing the test program or by + /// exiting with a failure. + /// + /// \param test_program The test program to execute. + /// \param vars User-provided variables to pass to the test program. + void + exec_list(const model::test_program& test_program, + const config::properties_map& vars) + const UTILS_NORETURN + { + const std::string name = test_program.absolute_path().leaf_name(); + + std::cerr << name; + std::cerr.flush(); + if (name == "check_i_exist") { + if (fs::exists(test_program.absolute_path())) { + std::cout << "found\n"; + do_exit(EXIT_SUCCESS); + } else { + std::cout << "not_found\n"; + do_exit(EXIT_FAILURE); + } + } else if (name == "empty") { + do_exit(EXIT_SUCCESS); + } else if (name == "misbehave") { + utils::abort_without_coredump(); + } else if (name == "timeout") { + std::cout << "sleeping\n"; + std::cout.flush(); + ::sleep(100); + utils::abort_without_coredump(); + } else if (name == "vars") { + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + std::cout << F("%s_%s\n") % (*iter).first % (*iter).second; + } + do_exit(15); + } else { + std::abort(); + } + } + + /// Computes the test cases list of a test program. + /// + /// \param status The termination status of the subprocess used to execute + /// the exec_test() method or none if the test timed out. + /// \param stdout_path Path to the file containing the stdout of the test. + /// \param stderr_path Path to the file containing the stderr of the test. + /// + /// \return A list of test cases. + model::test_cases_map + parse_list(const optional< process::status >& status, + const fs::path& stdout_path, + const fs::path& stderr_path) const + { + const std::string name = utils::read_file(stderr_path); + if (name == "check_i_exist") { + ATF_REQUIRE(status.get().exited()); + ATF_REQUIRE_EQ(EXIT_SUCCESS, status.get().exitstatus()); + } else if (name == "empty") { + ATF_REQUIRE(status.get().exited()); + ATF_REQUIRE_EQ(EXIT_SUCCESS, status.get().exitstatus()); + } else if (name == "misbehave") { + throw std::runtime_error("misbehaved in parse_list"); + } else if (name == "timeout") { + ATF_REQUIRE(!status); + } else if (name == "vars") { + ATF_REQUIRE(status.get().exited()); + ATF_REQUIRE_EQ(15, status.get().exitstatus()); + } else { + ATF_FAIL("Invalid stderr contents; got " + name); + } + + model::test_cases_map_builder test_cases_builder; + + std::ifstream input(stdout_path.c_str()); + ATF_REQUIRE(input); + std::string line; + while (std::getline(input, line).good()) { + test_cases_builder.add(line); + } + + return test_cases_builder.build(); + } + + /// Executes a test case of the test program. + /// + /// This method is intended to be called within a subprocess and is expected + /// to terminate execution either by exec(2)ing the test program or by + /// exiting with a failure. + /// + /// \param test_program The test program to execute. + /// \param test_case_name Name of the test case to invoke. + /// \param vars User-provided variables to pass to the test program. + /// \param control_directory Directory where the interface may place control + /// files. + void + exec_test(const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars, + const fs::path& control_directory) const + { + const fs::path cookie = control_directory / "exec_test_was_called"; + std::ofstream control_file(cookie.c_str()); + if (!control_file) { + std::cerr << "Failed to create " << cookie << '\n'; + std::abort(); + } + control_file << test_case_name; + control_file.close(); + + if (test_case_name == "check_i_exist") { + do_exit(fs::exists(test_program.absolute_path()) ? 0 : 1); + } else if (starts_with(test_case_name, "cleanup_timeout")) { + exec_exit(EXIT_SUCCESS); + } else if (starts_with(test_case_name, "create_files_and_fail")) { + exec_create_files_and_fail(); + } else if (test_case_name == "delete_all") { + exec_delete_all(); + } else if (starts_with(test_case_name, "exit ")) { + exec_exit(suffix_to_int(test_case_name, "exit ")); + } else if (starts_with(test_case_name, "fail")) { + exec_fail(); + } else if (starts_with(test_case_name, "fail_body_fail_cleanup")) { + exec_fail(); + } else if (starts_with(test_case_name, "fail_body_pass_cleanup")) { + exec_fail(); + } else if (starts_with(test_case_name, "pass_body_fail_cleanup")) { + exec_exit(EXIT_SUCCESS); + } else if (starts_with(test_case_name, "print_params")) { + exec_print_params(test_program, test_case_name, vars); + } else if (starts_with(test_case_name, "skip_body_pass_cleanup")) { + exec_exit(EXIT_SUCCESS); + } else { + std::cerr << "Unknown test case " << test_case_name << '\n'; + std::abort(); + } + } + + /// Executes a test cleanup routine of the test program. + /// + /// This method is intended to be called within a subprocess and is expected + /// to terminate execution either by exec(2)ing the test program or by + /// exiting with a failure. + /// + /// \param test_case_name Name of the test case to invoke. + void + exec_cleanup(const model::test_program& /* test_program */, + const std::string& test_case_name, + const config::properties_map& /* vars */, + const fs::path& /* control_directory */) const + { + std::cout << "exec_cleanup was called\n"; + std::cout.flush(); + + if (starts_with(test_case_name, "cleanup_timeout")) { + ::sleep(100); + std::abort(); + } else if (starts_with(test_case_name, "fail_body_fail_cleanup")) { + exec_fail(); + } else if (starts_with(test_case_name, "fail_body_pass_cleanup")) { + exec_exit(EXIT_SUCCESS); + } else if (starts_with(test_case_name, "pass_body_fail_cleanup")) { + exec_fail(); + } else if (starts_with(test_case_name, "skip_body_pass_cleanup")) { + exec_exit(EXIT_SUCCESS); + } else { + std::cerr << "Should not have been called for a test without " + "a cleanup routine" << '\n'; + std::abort(); + } + } + + /// Computes the result of a test case based on its termination status. + /// + /// \param status The termination status of the subprocess used to execute + /// the exec_test() method or none if the test timed out. + /// \param control_directory Path to the directory where the interface may + /// have placed control files. + /// \param stdout_path Path to the file containing the stdout of the test. + /// \param stderr_path Path to the file containing the stderr of the test. + /// + /// \return A test result. + model::test_result + compute_result(const optional< process::status >& status, + const fs::path& control_directory, + const fs::path& stdout_path, + const fs::path& stderr_path) const + { + // Do not use any ATF_* macros here. Some of the tests below invoke + // this code in a subprocess, and terminating such subprocess due to a + // failed ATF_* macro yields mysterious failures that are incredibly + // hard to debug. (Case in point: the signal_handling test is racy by + // nature, and the test run by exec_test() above may not have created + // the cookie we expect below. We don't want to "silently" exit if the + // file is not there.) + + if (!status) { + return model::test_result(model::test_result_broken, + "Timed out"); + } + + if (status.get().exited()) { + // Only sanity-check the work directory-related parameters in case + // of a clean exit. In all other cases, there is no guarantee that + // these were ever created. + const fs::path cookie = control_directory / "exec_test_was_called"; + if (!atf::utils::file_exists(cookie.str())) { + return model::test_result( + model::test_result_broken, + "compute_result's control_directory does not seem to point " + "to the right location"); + } + const std::string test_case_name = utils::read_file(cookie); + + if (!atf::utils::file_exists(stdout_path.str())) { + return model::test_result( + model::test_result_broken, + "compute_result's stdout_path does not exist"); + } + if (!atf::utils::file_exists(stderr_path.str())) { + return model::test_result( + model::test_result_broken, + "compute_result's stderr_path does not exist"); + } + + if (test_case_name == "skip_body_pass_cleanup") { + return model::test_result( + model::test_result_skipped, + F("Exit %s") % status.get().exitstatus()); + } else { + return model::test_result( + model::test_result_passed, + F("Exit %s") % status.get().exitstatus()); + } + } else { + return model::test_result( + model::test_result_failed, + F("Signal %s") % status.get().termsig()); + } + } +}; + + +} // anonymous namespace + + +/// Runs list_tests on the scheduler and returns the results. +/// +/// \param test_name The name of the test supported by our exec_list function. +/// \param user_config Optional user settings for the test. +/// +/// \return The loaded list of test cases. +static model::test_cases_map +check_integration_list(const char* test_name, const fs::path root, + const config::tree& user_config = engine::empty_config()) +{ + const model::test_program program = model::test_program_builder( + "mock", fs::path(test_name), root, "the-suite") + .build(); + + scheduler::scheduler_handle handle = scheduler::setup(); + const model::test_cases_map test_cases = handle.list_tests(&program, + user_config); + handle.cleanup(); + + return test_cases; +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_some); +ATF_TEST_CASE_BODY(integration__list_some) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.first", "test"); + user_config.set_string("test_suites.the-suite.second", "TEST"); + user_config.set_string("test_suites.abc.unused", "unused"); + + const model::test_cases_map test_cases = check_integration_list( + "vars", fs::path("."), user_config); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("first_test").add("second_TEST").build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_check_paths); +ATF_TEST_CASE_BODY(integration__list_check_paths) +{ + fs::mkdir_p(fs::path("dir1/dir2/dir3"), 0755); + atf::utils::create_file("dir1/dir2/dir3/check_i_exist", ""); + + const model::test_cases_map test_cases = check_integration_list( + "dir2/dir3/check_i_exist", fs::path("dir1")); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("found").build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_timeout); +ATF_TEST_CASE_BODY(integration__list_timeout) +{ + scheduler::list_timeout = datetime::delta(1, 0); + const model::test_cases_map test_cases = check_integration_list( + "timeout", fs::path(".")); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("sleeping").build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_fail); +ATF_TEST_CASE_BODY(integration__list_fail) +{ + const model::test_cases_map test_cases = check_integration_list( + "misbehave", fs::path(".")); + + ATF_REQUIRE_EQ(1, test_cases.size()); + const model::test_case& test_case = test_cases.begin()->second; + ATF_REQUIRE_EQ("__test_cases_list__", test_case.name()); + ATF_REQUIRE(test_case.fake_result()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_broken, + "misbehaved in parse_list"), + test_case.fake_result().get()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_empty); +ATF_TEST_CASE_BODY(integration__list_empty) +{ + const model::test_cases_map test_cases = check_integration_list( + "empty", fs::path(".")); + + ATF_REQUIRE_EQ(1, test_cases.size()); + const model::test_case& test_case = test_cases.begin()->second; + ATF_REQUIRE_EQ("__test_cases_list__", test_case.name()); + ATF_REQUIRE(test_case.fake_result()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_broken, + "Empty test cases list"), + test_case.fake_result().get()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__run_one); +ATF_TEST_CASE_BODY(integration__run_one) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("exit 41").build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + const scheduler::exec_handle exec_handle = handle.spawn_test( + program, "exit 41", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(exec_handle, result_handle->original_pid()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 41"), + test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__run_many); +ATF_TEST_CASE_BODY(integration__run_many) +{ + static const std::size_t num_test_programs = 30; + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + // We mess around with the "current time" below, so make sure the tests do + // not spuriously exceed their deadline by bumping it to a large number. + const model::metadata infinite_timeout = model::metadata_builder() + .set_timeout(datetime::delta(1000000L, 0)).build(); + + std::size_t total_tests = 0; + std::map< scheduler::exec_handle, model::test_program_ptr > + exp_test_programs; + std::map< scheduler::exec_handle, std::string > exp_test_case_names; + std::map< scheduler::exec_handle, datetime::timestamp > exp_start_times; + std::map< scheduler::exec_handle, int > exp_exit_statuses; + for (std::size_t i = 0; i < num_test_programs; ++i) { + const std::string test_case_0 = F("exit %s") % (i * 3 + 0); + const std::string test_case_1 = F("exit %s") % (i * 3 + 1); + const std::string test_case_2 = F("exit %s") % (i * 3 + 2); + + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path(F("program-%s") % i), + fs::current_path(), "the-suite") + .set_metadata(infinite_timeout) + .add_test_case(test_case_0) + .add_test_case(test_case_1) + .add_test_case(test_case_2) + .build_ptr(); + + const datetime::timestamp start_time = datetime::timestamp::from_values( + 2014, 12, 8, 9, 40, 0, i); + + scheduler::exec_handle exec_handle; + + datetime::set_mock_now(start_time); + exec_handle = handle.spawn_test(program, test_case_0, user_config); + exp_test_programs.insert(std::make_pair(exec_handle, program)); + exp_test_case_names.insert(std::make_pair(exec_handle, test_case_0)); + exp_start_times.insert(std::make_pair(exec_handle, start_time)); + exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3)); + ++total_tests; + + datetime::set_mock_now(start_time); + exec_handle = handle.spawn_test(program, test_case_1, user_config); + exp_test_programs.insert(std::make_pair(exec_handle, program)); + exp_test_case_names.insert(std::make_pair(exec_handle, test_case_1)); + exp_start_times.insert(std::make_pair(exec_handle, start_time)); + exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3 + 1)); + ++total_tests; + + datetime::set_mock_now(start_time); + exec_handle = handle.spawn_test(program, test_case_2, user_config); + exp_test_programs.insert(std::make_pair(exec_handle, program)); + exp_test_case_names.insert(std::make_pair(exec_handle, test_case_2)); + exp_start_times.insert(std::make_pair(exec_handle, start_time)); + exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3 + 2)); + ++total_tests; + } + + for (std::size_t i = 0; i < total_tests; ++i) { + const datetime::timestamp end_time = datetime::timestamp::from_values( + 2014, 12, 8, 9, 50, 10, i); + datetime::set_mock_now(end_time); + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + + const scheduler::exec_handle exec_handle = + result_handle->original_pid(); + + const model::test_program_ptr test_program = exp_test_programs.find( + exec_handle)->second; + const std::string& test_case_name = exp_test_case_names.find( + exec_handle)->second; + const datetime::timestamp& start_time = exp_start_times.find( + exec_handle)->second; + const int exit_status = exp_exit_statuses.find(exec_handle)->second; + + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, + F("Exit %s") % exit_status), + test_result_handle->test_result()); + + ATF_REQUIRE_EQ(test_program, test_result_handle->test_program()); + ATF_REQUIRE_EQ(test_case_name, test_result_handle->test_case_name()); + + ATF_REQUIRE_EQ(start_time, result_handle->start_time()); + ATF_REQUIRE_EQ(end_time, result_handle->end_time()); + + result_handle->cleanup(); + + ATF_REQUIRE(!atf::utils::file_exists( + result_handle->stdout_file().str())); + ATF_REQUIRE(!atf::utils::file_exists( + result_handle->stderr_file().str())); + ATF_REQUIRE(!atf::utils::file_exists( + result_handle->work_directory().str())); + + result_handle.reset(); + } + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__run_check_paths); +ATF_TEST_CASE_BODY(integration__run_check_paths) +{ + fs::mkdir_p(fs::path("dir1/dir2/dir3"), 0755); + atf::utils::create_file("dir1/dir2/dir3/program", ""); + + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("dir2/dir3/program"), fs::path("dir1"), "the-suite") + .add_test_case("check_i_exist").build_ptr(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "check_i_exist", engine::default_config()); + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"), + test_result_handle->test_result()); + + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__parameters_and_output); +ATF_TEST_CASE_BODY(integration__parameters_and_output) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("print_params").build_ptr(); + + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.one", "first variable"); + user_config.set_string("test_suites.the-suite.two", "second variable"); + + scheduler::scheduler_handle handle = scheduler::setup(); + + const scheduler::exec_handle exec_handle = handle.spawn_test( + program, "print_params", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + + ATF_REQUIRE_EQ(exec_handle, result_handle->original_pid()); + ATF_REQUIRE_EQ(program, test_result_handle->test_program()); + ATF_REQUIRE_EQ("print_params", test_result_handle->test_case_name()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"), + test_result_handle->test_result()); + + const fs::path stdout_file = result_handle->stdout_file(); + ATF_REQUIRE(atf::utils::compare_file( + stdout_file.str(), + "Test program: the-program\n" + "Test case: print_params\n" + "one=first variable\n" + "two=second variable\n")); + const fs::path stderr_file = result_handle->stderr_file(); + ATF_REQUIRE(atf::utils::compare_file( + stderr_file.str(), "stderr: print_params\n")); + + result_handle->cleanup(); + ATF_REQUIRE(!fs::exists(stdout_file)); + ATF_REQUIRE(!fs::exists(stderr_file)); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__fake_result); +ATF_TEST_CASE_BODY(integration__fake_result) +{ + const model::test_result fake_result(model::test_result_skipped, + "Some fake details"); + + model::test_cases_map test_cases; + test_cases.insert(model::test_cases_map::value_type( + "__fake__", model::test_case("__fake__", "ABC", fake_result))); + + const model::test_program_ptr program(new model::test_program( + "mock", fs::path("the-program"), fs::current_path(), "the-suite", + model::metadata_builder().build(), test_cases)); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "__fake__", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(fake_result, test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__head_skips); +ATF_TEST_CASE_BODY(integration__cleanup__head_skips) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("skip_me", + model::metadata_builder() + .add_required_config("variable-that-does-not-exist") + .set_has_cleanup(true) + .build()) + .build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "skip_me", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(model::test_result( + model::test_result_skipped, + "Required configuration property " + "'variable-that-does-not-exist' not defined"), + test_result_handle->test_result()); + ATF_REQUIRE(!atf::utils::grep_file("exec_cleanup was called", + result_handle->stdout_file().str())); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +/// Runs a test to verify the behavior of cleanup routines. +/// +/// \param test_case The name of the test case to invoke. +/// \param exp_result The expected test result of the execution. +static void +do_cleanup_test(const char* test_case, + const model::test_result& exp_result) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case(test_case) + .set_metadata(model::metadata_builder().set_has_cleanup(true).build()) + .build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, test_case, user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); + ATF_REQUIRE(atf::utils::compare_file( + result_handle->stdout_file().str(), + "exec_cleanup was called\n")); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_skips); +ATF_TEST_CASE_BODY(integration__cleanup__body_skips) +{ + do_cleanup_test( + "skip_body_pass_cleanup", + model::test_result(model::test_result_skipped, "Exit 0")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_bad__cleanup_ok); +ATF_TEST_CASE_BODY(integration__cleanup__body_bad__cleanup_ok) +{ + do_cleanup_test( + "fail_body_pass_cleanup", + model::test_result(model::test_result_failed, "Signal 15")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_ok__cleanup_bad); +ATF_TEST_CASE_BODY(integration__cleanup__body_ok__cleanup_bad) +{ + do_cleanup_test( + "pass_body_fail_cleanup", + model::test_result(model::test_result_broken, "Test case cleanup " + "did not terminate successfully")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_bad__cleanup_bad); +ATF_TEST_CASE_BODY(integration__cleanup__body_bad__cleanup_bad) +{ + do_cleanup_test( + "fail_body_fail_cleanup", + model::test_result(model::test_result_failed, "Signal 15")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__timeout); +ATF_TEST_CASE_BODY(integration__cleanup__timeout) +{ + scheduler::cleanup_timeout = datetime::delta(1, 0); + do_cleanup_test( + "cleanup_timeout", + model::test_result(model::test_result_broken, "Test case cleanup " + "timed out")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__check_requirements); +ATF_TEST_CASE_BODY(integration__check_requirements) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("exit 12") + .set_metadata(model::metadata_builder() + .add_required_config("abcde").build()) + .build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "exit 12", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(model::test_result( + model::test_result_skipped, + "Required configuration property 'abcde' not defined"), + test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__stacktrace); +ATF_TEST_CASE_BODY(integration__stacktrace) +{ + utils::prepare_coredump_test(this); + + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("unknown-dumps-core").build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "unknown-dumps-core", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_failed, + F("Signal %s") % SIGABRT), + test_result_handle->test_result()); + ATF_REQUIRE(!atf::utils::grep_file("attempting to gather stack trace", + result_handle->stdout_file().str())); + ATF_REQUIRE( atf::utils::grep_file("attempting to gather stack trace", + result_handle->stderr_file().str())); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +/// Runs a test to verify the dumping of the list of existing files on failure. +/// +/// \param test_case The name of the test case to invoke. +/// \param exp_stderr Expected contents of stderr. +static void +do_check_list_files_on_failure(const char* test_case, const char* exp_stderr) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case(test_case).build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, test_case, user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + atf::utils::cat_file(result_handle->stdout_file().str(), "child stdout: "); + ATF_REQUIRE(atf::utils::compare_file(result_handle->stdout_file().str(), + "")); + atf::utils::cat_file(result_handle->stderr_file().str(), "child stderr: "); + ATF_REQUIRE(atf::utils::compare_file(result_handle->stderr_file().str(), + exp_stderr)); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_files_on_failure__none); +ATF_TEST_CASE_BODY(integration__list_files_on_failure__none) +{ + do_check_list_files_on_failure("fail", "This should not be clobbered\n"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_files_on_failure__some); +ATF_TEST_CASE_BODY(integration__list_files_on_failure__some) +{ + do_check_list_files_on_failure( + "create_files_and_fail", + "This should not be clobbered\n" + "Files left in work directory after failure: " + "dir1, first file, second-file\n"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__prevent_clobbering_control_files); +ATF_TEST_CASE_BODY(integration__prevent_clobbering_control_files) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("delete_all").build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "delete_all", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"), + test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(debug_test); +ATF_TEST_CASE_BODY(debug_test) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("print_params").build_ptr(); + + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.one", "first variable"); + user_config.set_string("test_suites.the-suite.two", "second variable"); + + scheduler::scheduler_handle handle = scheduler::setup(); + + const fs::path stdout_file("custom-stdout.txt"); + const fs::path stderr_file("custom-stderr.txt"); + + scheduler::result_handle_ptr result_handle = handle.debug_test( + program, "print_params", user_config, stdout_file, stderr_file); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + + ATF_REQUIRE_EQ(program, test_result_handle->test_program()); + ATF_REQUIRE_EQ("print_params", test_result_handle->test_case_name()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"), + test_result_handle->test_result()); + + // The original output went to a file. It's only an artifact of + // debug_test() that we later get a copy in our own files. + ATF_REQUIRE(stdout_file != result_handle->stdout_file()); + ATF_REQUIRE(stderr_file != result_handle->stderr_file()); + + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); + + ATF_REQUIRE(atf::utils::compare_file( + stdout_file.str(), + "Test program: the-program\n" + "Test case: print_params\n" + "one=first variable\n" + "two=second variable\n")); + ATF_REQUIRE(atf::utils::compare_file( + stderr_file.str(), "stderr: print_params\n")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(ensure_valid_interface); +ATF_TEST_CASE_BODY(ensure_valid_interface) +{ + scheduler::ensure_valid_interface("mock"); + + ATF_REQUIRE_THROW_RE(engine::error, "Unsupported test interface 'mock2'", + scheduler::ensure_valid_interface("mock2")); + scheduler::register_interface( + "mock2", std::shared_ptr< scheduler::interface >(new mock_interface())); + scheduler::ensure_valid_interface("mock2"); + + // Standard interfaces should not be present unless registered. + ATF_REQUIRE_THROW_RE(engine::error, "Unsupported test interface 'plain'", + scheduler::ensure_valid_interface("plain")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(registered_interface_names); +ATF_TEST_CASE_BODY(registered_interface_names) +{ + std::set< std::string > exp_names; + + exp_names.insert("mock"); + ATF_REQUIRE_EQ(exp_names, scheduler::registered_interface_names()); + + scheduler::register_interface( + "mock2", std::shared_ptr< scheduler::interface >(new mock_interface())); + exp_names.insert("mock2"); + ATF_REQUIRE_EQ(exp_names, scheduler::registered_interface_names()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(current_context); +ATF_TEST_CASE_BODY(current_context) +{ + const model::context context = scheduler::current_context(); + ATF_REQUIRE_EQ(fs::current_path(), context.cwd()); + ATF_REQUIRE(utils::getallenv() == context.env()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(generate_config__empty); +ATF_TEST_CASE_BODY(generate_config__empty) +{ + const config::tree user_config = engine::empty_config(); + + const config::properties_map exp_props; + + ATF_REQUIRE_EQ(exp_props, + scheduler::generate_config(user_config, "missing")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(generate_config__no_matches); +ATF_TEST_CASE_BODY(generate_config__no_matches) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("architecture", "foo"); + user_config.set_string("test_suites.one.var1", "value 1"); + + const config::properties_map exp_props; + + ATF_REQUIRE_EQ(exp_props, + scheduler::generate_config(user_config, "two")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(generate_config__some_matches); +ATF_TEST_CASE_BODY(generate_config__some_matches) +{ + std::vector< passwd::user > mock_users; + mock_users.push_back(passwd::user("nobody", 1234, 5678)); + passwd::set_mock_users_for_testing(mock_users); + + config::tree user_config = engine::empty_config(); + user_config.set_string("architecture", "foo"); + user_config.set_string("unprivileged_user", "nobody"); + user_config.set_string("test_suites.one.var1", "value 1"); + user_config.set_string("test_suites.two.var2", "value 2"); + + config::properties_map exp_props; + exp_props["unprivileged-user"] = "nobody"; + exp_props["var1"] = "value 1"; + + ATF_REQUIRE_EQ(exp_props, + scheduler::generate_config(user_config, "one")); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + scheduler::register_interface( + "mock", std::shared_ptr< scheduler::interface >(new mock_interface())); + + ATF_ADD_TEST_CASE(tcs, integration__list_some); + ATF_ADD_TEST_CASE(tcs, integration__list_check_paths); + ATF_ADD_TEST_CASE(tcs, integration__list_timeout); + ATF_ADD_TEST_CASE(tcs, integration__list_fail); + ATF_ADD_TEST_CASE(tcs, integration__list_empty); + + ATF_ADD_TEST_CASE(tcs, integration__run_one); + ATF_ADD_TEST_CASE(tcs, integration__run_many); + + ATF_ADD_TEST_CASE(tcs, integration__run_check_paths); + ATF_ADD_TEST_CASE(tcs, integration__parameters_and_output); + + ATF_ADD_TEST_CASE(tcs, integration__fake_result); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__head_skips); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_skips); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_ok__cleanup_bad); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_bad__cleanup_ok); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_bad__cleanup_bad); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__timeout); + ATF_ADD_TEST_CASE(tcs, integration__check_requirements); + ATF_ADD_TEST_CASE(tcs, integration__stacktrace); + ATF_ADD_TEST_CASE(tcs, integration__list_files_on_failure__none); + ATF_ADD_TEST_CASE(tcs, integration__list_files_on_failure__some); + ATF_ADD_TEST_CASE(tcs, integration__prevent_clobbering_control_files); + + ATF_ADD_TEST_CASE(tcs, debug_test); + + ATF_ADD_TEST_CASE(tcs, ensure_valid_interface); + ATF_ADD_TEST_CASE(tcs, registered_interface_names); + + ATF_ADD_TEST_CASE(tcs, current_context); + + ATF_ADD_TEST_CASE(tcs, generate_config__empty); + ATF_ADD_TEST_CASE(tcs, generate_config__no_matches); + ATF_ADD_TEST_CASE(tcs, generate_config__some_matches); +} diff --git a/engine/tap.cpp b/engine/tap.cpp new file mode 100644 index 000000000000..85e23857f5b7 --- /dev/null +++ b/engine/tap.cpp @@ -0,0 +1,191 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/tap.hpp" + +extern "C" { +#include <unistd.h> +} + +#include <cstdlib> + +#include "engine/exceptions.hpp" +#include "engine/tap_parser.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/defs.hpp" +#include "utils/env.hpp" +#include "utils/format/macros.hpp" +#include "utils/optional.ipp" +#include "utils/process/operations.hpp" +#include "utils/process/status.hpp" +#include "utils/sanity.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace process = utils::process; + +using utils::optional; + + +namespace { + + +/// Computes the result of a TAP test program termination. +/// +/// Timeouts and bad TAP data must be handled by the caller. Here we assume +/// that we have been able to successfully parse the TAP output. +/// +/// \param summary Parsed TAP data for the test program. +/// \param status Exit status of the test program. +/// +/// \return A test result. +static model::test_result +tap_to_result(const engine::tap_summary& summary, + const process::status& status) +{ + if (summary.bailed_out()) { + return model::test_result(model::test_result_failed, "Bailed out"); + } + + if (summary.plan() == engine::all_skipped_plan) { + return model::test_result(model::test_result_skipped, + summary.all_skipped_reason()); + } + + if (summary.not_ok_count() == 0) { + if (status.exitstatus() == EXIT_SUCCESS) { + return model::test_result(model::test_result_passed); + } else { + return model::test_result( + model::test_result_broken, + F("Dubious test program: reported all tests as passed " + "but returned exit code %s") % status.exitstatus()); + } + } else { + const std::size_t total = summary.ok_count() + summary.not_ok_count(); + return model::test_result(model::test_result_failed, + F("%s of %s tests failed") % + summary.not_ok_count() % total); + } +} + + +} // anonymous namespace + + +/// Executes a test program's list operation. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +void +engine::tap_interface::exec_list( + const model::test_program& /* test_program */, + const config::properties_map& /* vars */) const +{ + ::_exit(EXIT_SUCCESS); +} + + +/// Computes the test cases list of a test program. +/// +/// \return A list of test cases. +model::test_cases_map +engine::tap_interface::parse_list( + const optional< process::status >& /* status */, + const fs::path& /* stdout_path */, + const fs::path& /* stderr_path */) const +{ + return model::test_cases_map_builder().add("main").build(); +} + + +/// Executes a test case of the test program. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +/// +/// \param test_program The test program to execute. +/// \param test_case_name Name of the test case to invoke. +/// \param vars User-provided variables to pass to the test program. +void +engine::tap_interface::exec_test( + const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars, + const fs::path& /* control_directory */) const +{ + PRE(test_case_name == "main"); + + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + utils::setenv(F("TEST_ENV_%s") % (*iter).first, (*iter).second); + } + + process::args_vector args; + process::exec(test_program.absolute_path(), args); +} + + +/// Computes the result of a test case based on its termination status. +/// +/// \param status The termination status of the subprocess used to execute +/// the exec_test() method or none if the test timed out. +/// \param stdout_path Path to the file containing the stdout of the test. +/// +/// \return A test result. +model::test_result +engine::tap_interface::compute_result( + const optional< process::status >& status, + const fs::path& /* control_directory */, + const fs::path& stdout_path, + const fs::path& /* stderr_path */) const +{ + if (!status) { + return model::test_result(model::test_result_broken, + "Test case timed out"); + } else { + if (status.get().signaled()) { + return model::test_result( + model::test_result_broken, + F("Received signal %s") % status.get().termsig()); + } else { + try { + const tap_summary summary = parse_tap_output(stdout_path); + return tap_to_result(summary, status.get()); + } catch (const load_error& e) { + return model::test_result( + model::test_result_broken, + F("TAP test program yielded invalid data: %s") % e.what()); + } + } + } +} diff --git a/engine/tap.hpp b/engine/tap.hpp new file mode 100644 index 000000000000..b46bf28f0240 --- /dev/null +++ b/engine/tap.hpp @@ -0,0 +1,67 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/tap.hpp +/// Execution engine for test programs that output the TAP protocol. + +#if !defined(ENGINE_TAP_HPP) +#define ENGINE_TAP_HPP + +#include "engine/scheduler.hpp" + +namespace engine { + + +/// Implementation of the scheduler interface for tap test programs. +class tap_interface : public engine::scheduler::interface { +public: + void exec_list(const model::test_program&, + const utils::config::properties_map&) const UTILS_NORETURN; + + model::test_cases_map parse_list( + const utils::optional< utils::process::status >&, + const utils::fs::path&, + const utils::fs::path&) const; + + void exec_test(const model::test_program&, const std::string&, + const utils::config::properties_map&, + const utils::fs::path&) const + UTILS_NORETURN; + + model::test_result compute_result( + const utils::optional< utils::process::status >&, + const utils::fs::path&, + const utils::fs::path&, + const utils::fs::path&) const; +}; + + +} // namespace engine + + +#endif // !defined(ENGINE_TAP_HPP) diff --git a/engine/tap_helpers.cpp b/engine/tap_helpers.cpp new file mode 100644 index 000000000000..4f9505c78dec --- /dev/null +++ b/engine/tap_helpers.cpp @@ -0,0 +1,202 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include <sys/stat.h> + +#include <unistd.h> + +extern char** environ; +} + +#include <cstdlib> +#include <cstring> +#include <fstream> +#include <iostream> + +#include "utils/env.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/test_utils.ipp" + +namespace fs = utils::fs; + + +namespace { + + +/// Logs an error message and exits the test with an error code. +/// +/// \param str The error message to log. +static void +fail(const std::string& str) +{ + std::cerr << str << '\n'; + std::exit(EXIT_FAILURE); +} + + +/// A test scenario that validates the TEST_ENV_* variables. +static void +test_check_configuration_variables(void) +{ + std::set< std::string > vars; + char** iter; + for (iter = environ; *iter != NULL; ++iter) { + if (std::strstr(*iter, "TEST_ENV_") == *iter) { + vars.insert(*iter); + } + } + + std::set< std::string > exp_vars; + exp_vars.insert("TEST_ENV_first=some value"); + exp_vars.insert("TEST_ENV_second=some other value"); + if (vars == exp_vars) { + std::cout << "1..1\n" + << "ok 1\n"; + } else { + std::cout << "1..1\n" + << "not ok 1\n" + << F(" Expected: %s\nFound: %s\n") % exp_vars % vars; + } +} + + +/// A test scenario that crashes. +static void +test_crash(void) +{ + utils::abort_without_coredump(); +} + + +/// A test scenario that reports some tests as failed. +static void +test_fail(void) +{ + std::cout << "1..5\n" + << "ok 1 - This is good!\n" + << "not ok 2\n" + << "ok 3 - TODO Consider this as passed\n" + << "ok 4\n" + << "not ok 5\n"; +} + + +/// A test scenario that passes. +static void +test_pass(void) +{ + std::cout << "1..4\n" + << "ok 1 - This is good!\n" + << "non-result data\n" + << "ok 2 - SKIP Consider this as passed\n" + << "ok 3 - TODO Consider this as passed\n" + << "ok 4\n"; +} + + +/// A test scenario that passes but then exits with non-zero. +static void +test_pass_but_exit_failure(void) +{ + std::cout << "1..2\n" + << "ok 1\n" + << "ok 2\n"; + std::exit(70); +} + + +/// A test scenario that times out. +/// +/// Note that the timeout is defined in the Kyuafile, as the TAP interface has +/// no means for test programs to specify this by themselves. +static void +test_timeout(void) +{ + std::cout << "1..2\n" + << "ok 1\n"; + + ::sleep(10); + const fs::path control_dir = fs::path(utils::getenv("CONTROL_DIR").get()); + std::ofstream file((control_dir / "cookie").c_str()); + if (!file) + fail("Failed to create the control cookie"); + file.close(); +} + + +} // anonymous namespace + + +/// Entry point to the test program. +/// +/// The caller can select which test scenario to run by modifying the program's +/// basename on disk (either by a copy or by a hard link). +/// +/// \todo It may be worth to split this binary into separate, smaller binaries, +/// one for every "test scenario". We use this program as a dispatcher for +/// different "main"s, the only reason being to keep the amount of helper test +/// programs to a minimum. However, putting this each function in its own +/// binary could simplify many other things. +/// +/// \param argc The number of CLI arguments. +/// \param argv The CLI arguments themselves. These are not used because +/// Kyua will not pass any arguments to the plain test program. +int +main(int argc, char** argv) +{ + if (argc != 1) { + std::cerr << "No arguments allowed; select the test scenario with the " + "program's basename\n"; + return EXIT_FAILURE; + } + + const std::string& test_scenario = fs::path(argv[0]).leaf_name(); + + if (test_scenario == "check_configuration_variables") + test_check_configuration_variables(); + else if (test_scenario == "crash") + test_crash(); + else if (test_scenario == "fail") + test_fail(); + else if (test_scenario == "pass") + test_pass(); + else if (test_scenario == "pass_but_exit_failure") + test_pass_but_exit_failure(); + else if (test_scenario == "timeout") + test_timeout(); + else { + std::cerr << "Unknown test scenario\n"; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/engine/tap_parser.cpp b/engine/tap_parser.cpp new file mode 100644 index 000000000000..d41328534fad --- /dev/null +++ b/engine/tap_parser.cpp @@ -0,0 +1,438 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/tap_parser.hpp" + +#include <fstream> + +#include "engine/exceptions.hpp" +#include "utils/format/macros.hpp" +#include "utils/noncopyable.hpp" +#include "utils/optional.ipp" +#include "utils/sanity.hpp" +#include "utils/text/exceptions.hpp" +#include "utils/text/operations.ipp" +#include "utils/text/regex.hpp" + +namespace fs = utils::fs; +namespace text = utils::text; + +using utils::optional; + + +/// TAP plan representing all tests being skipped. +const engine::tap_plan engine::all_skipped_plan(1, 0); + + +namespace { + + +/// Implementation of the TAP parser. +/// +/// This is a class only to simplify keeping global constant values around (like +/// prebuilt regular expressions). +class tap_parser : utils::noncopyable { + /// Regular expression to match plan lines. + text::regex _plan_regex; + + /// Regular expression to match a TODO and extract the reason. + text::regex _todo_regex; + + /// Regular expression to match a SKIP and extract the reason. + text::regex _skip_regex; + + /// Regular expression to match a single test result. + text::regex _result_regex; + + /// Checks if a line contains a TAP plan and extracts its data. + /// + /// \param line The line to try to parse. + /// \param [in,out] out_plan Used to store the found plan, if any. The same + /// output variable should be given to all calls to this function so + /// that duplicate plan entries can be discovered. + /// \param [out] out_all_skipped_reason Used to store the reason for all + /// tests being skipped, if any. If this is set to a non-empty value, + /// then the out_plan is set to 1..0. + /// + /// \return True if the line matched a plan; false otherwise. + /// + /// \throw engine::format_error If the input is invalid. + /// \throw text::error If the input is invalid. + bool + try_parse_plan(const std::string& line, + optional< engine::tap_plan >& out_plan, + std::string& out_all_skipped_reason) + { + const text::regex_matches plan_matches = _plan_regex.match(line); + if (!plan_matches) + return false; + const engine::tap_plan plan( + text::to_type< std::size_t >(plan_matches.get(1)), + text::to_type< std::size_t >(plan_matches.get(2))); + + if (out_plan) + throw engine::format_error( + F("Found duplicate plan %s..%s (saw %s..%s earlier)") % + plan.first % plan.second % + out_plan.get().first % out_plan.get().second); + + std::string all_skipped_reason; + const text::regex_matches skip_matches = _skip_regex.match(line); + if (skip_matches) { + if (plan != engine::all_skipped_plan) { + throw engine::format_error(F("Skipped plan must be %s..%s") % + engine::all_skipped_plan.first % + engine::all_skipped_plan.second); + } + all_skipped_reason = skip_matches.get(2); + if (all_skipped_reason.empty()) + all_skipped_reason = "No reason specified"; + } else { + if (plan.first > plan.second) + throw engine::format_error(F("Found reversed plan %s..%s") % + plan.first % plan.second); + } + + INV(!out_plan); + out_plan = plan; + out_all_skipped_reason = all_skipped_reason; + + POST(out_plan); + POST(out_all_skipped_reason.empty() || + out_plan.get() == engine::all_skipped_plan); + + return true; + } + + /// Checks if a line contains a TAP test result and extracts its data. + /// + /// \param line The line to try to parse. + /// \param [in,out] out_ok_count Accumulator for 'ok' results. + /// \param [in,out] out_not_ok_count Accumulator for 'not ok' results. + /// \param [out] out_bailed_out Set to true if the test bailed out. + /// + /// \return True if the line matched a result; false otherwise. + /// + /// \throw engine::format_error If the input is invalid. + /// \throw text::error If the input is invalid. + bool + try_parse_result(const std::string& line, std::size_t& out_ok_count, + std::size_t& out_not_ok_count, bool& out_bailed_out) + { + PRE(!out_bailed_out); + + const text::regex_matches result_matches = _result_regex.match(line); + if (result_matches) { + if (result_matches.get(1) == "ok") { + ++out_ok_count; + } else { + INV(result_matches.get(1) == "not ok"); + if (_todo_regex.match(line) || _skip_regex.match(line)) { + ++out_ok_count; + } else { + ++out_not_ok_count; + } + } + return true; + } else { + if (line.find("Bail out!") == 0) { + out_bailed_out = true; + return true; + } else { + return false; + } + } + } + +public: + /// Sets up the TAP parser state. + tap_parser(void) : + _plan_regex(text::regex::compile("^([0-9]+)\\.\\.([0-9]+)", 2)), + _todo_regex(text::regex::compile("TODO[ \t]*(.*)$", 2, true)), + _skip_regex(text::regex::compile("(SKIP|Skipped:?)[ \t]*(.*)$", 2, + true)), + _result_regex(text::regex::compile("^(not ok|ok)[ \t-]+[0-9]*", 1)) + { + } + + /// Parses an input file containing TAP output. + /// + /// \param input The stream to read from. + /// + /// \return The results of the parsing in the form of a tap_summary object. + /// + /// \throw engine::format_error If there are any syntax errors in the input. + /// \throw text::error If there are any syntax errors in the input. + engine::tap_summary + parse(std::ifstream& input) + { + optional< engine::tap_plan > plan; + std::string all_skipped_reason; + bool bailed_out = false; + std::size_t ok_count = 0, not_ok_count = 0; + + std::string line; + while (!bailed_out && std::getline(input, line)) { + if (try_parse_result(line, ok_count, not_ok_count, bailed_out)) + continue; + (void)try_parse_plan(line, plan, all_skipped_reason); + } + + if (bailed_out) { + return engine::tap_summary::new_bailed_out(); + } else { + if (!plan) + throw engine::format_error( + "Output did not contain any TAP plan and the program did " + "not bail out"); + + if (plan.get() == engine::all_skipped_plan) { + return engine::tap_summary::new_all_skipped(all_skipped_reason); + } else { + const std::size_t exp_count = plan.get().second - + plan.get().first + 1; + const std::size_t actual_count = ok_count + not_ok_count; + if (exp_count != actual_count) { + throw engine::format_error( + "Reported plan differs from actual executed tests"); + } + return engine::tap_summary::new_results(plan.get(), ok_count, + not_ok_count); + } + } + } +}; + + +} // anonymous namespace + + +/// Constructs a TAP summary with the results of parsing a TAP output. +/// +/// \param bailed_out_ Whether the test program bailed out early or not. +/// \param plan_ The TAP plan. +/// \param all_skipped_reason_ The reason for skipping all tests, if any. +/// \param ok_count_ Number of 'ok' test results. +/// \param not_ok_count_ Number of 'not ok' test results. +engine::tap_summary::tap_summary(const bool bailed_out_, + const tap_plan& plan_, + const std::string& all_skipped_reason_, + const std::size_t ok_count_, + const std::size_t not_ok_count_) : + _bailed_out(bailed_out_), _plan(plan_), + _all_skipped_reason(all_skipped_reason_), + _ok_count(ok_count_), _not_ok_count(not_ok_count_) +{ +} + + +/// Constructs a TAP summary for a bailed out test program. +/// +/// \return The new tap_summary object. +engine::tap_summary +engine::tap_summary::new_bailed_out(void) +{ + return tap_summary(true, tap_plan(0, 0), "", 0, 0); +} + + +/// Constructs a TAP summary for a test program that skipped all tests. +/// +/// \param reason Textual reason describing why the tests were skipped. +/// +/// \return The new tap_summary object. +engine::tap_summary +engine::tap_summary::new_all_skipped(const std::string& reason) +{ + return tap_summary(false, all_skipped_plan, reason, 0, 0); +} + + +/// Constructs a TAP summary for a test program that reported results. +/// +/// \param plan_ The TAP plan. +/// \param ok_count_ Total number of 'ok' results. +/// \param not_ok_count_ Total number of 'not ok' results. +/// +/// \return The new tap_summary object. +engine::tap_summary +engine::tap_summary::new_results(const tap_plan& plan_, + const std::size_t ok_count_, + const std::size_t not_ok_count_) +{ + PRE((plan_.second - plan_.first + 1) == (ok_count_ + not_ok_count_)); + return tap_summary(false, plan_, "", ok_count_, not_ok_count_); +} + + +/// Checks whether the test program bailed out early or not. +/// +/// \return True if the test program aborted execution before completing. +bool +engine::tap_summary::bailed_out(void) const +{ + return _bailed_out; +} + + +/// Gets the TAP plan of the test program. +/// +/// \pre bailed_out() must be false. +/// +/// \return The TAP plan. If 1..0, then all_skipped_reason() will have some +/// contents. +const engine::tap_plan& +engine::tap_summary::plan(void) const +{ + PRE(!_bailed_out); + return _plan; +} + + +/// Gets the reason for skipping all the tests, if any. +/// +/// \pre bailed_out() must be false. +/// \pre plan() returns 1..0. +/// +/// \return The reason for skipping all the tests. +const std::string& +engine::tap_summary::all_skipped_reason(void) const +{ + PRE(!_bailed_out); + PRE(_plan == all_skipped_plan); + return _all_skipped_reason; +} + + +/// Gets the number of 'ok' test results. +/// +/// \pre bailed_out() must be false. +/// +/// \return The number of test results that reported 'ok'. +std::size_t +engine::tap_summary::ok_count(void) const +{ + PRE(!bailed_out()); + PRE(_all_skipped_reason.empty()); + return _ok_count; +} + + +/// Gets the number of 'not ok' test results. +/// +/// \pre bailed_out() must be false. +/// +/// \return The number of test results that reported 'not ok'. +std::size_t +engine::tap_summary::not_ok_count(void) const +{ + PRE(!_bailed_out); + PRE(_all_skipped_reason.empty()); + return _not_ok_count; +} + + +/// Checks two tap_summary objects for equality. +/// +/// \param other The object to compare this one to. +/// +/// \return True if the two objects are equal; false otherwise. +bool +engine::tap_summary::operator==(const tap_summary& other) const +{ + return (_bailed_out == other._bailed_out && + _plan == other._plan && + _all_skipped_reason == other._all_skipped_reason && + _ok_count == other._ok_count && + _not_ok_count == other._not_ok_count); +} + + +/// Checks two tap_summary objects for inequality. +/// +/// \param other The object to compare this one to. +/// +/// \return True if the two objects are different; false otherwise. +bool +engine::tap_summary::operator!=(const tap_summary& other) const +{ + return !(*this == other); +} + + +/// Formats a tap_summary into a stream. +/// +/// \param output The stream into which to inject the object. +/// \param summary The summary to format. +/// +/// \return The output stream. +std::ostream& +engine::operator<<(std::ostream& output, const tap_summary& summary) +{ + output << "tap_summary{"; + if (summary.bailed_out()) { + output << "bailed_out=true"; + } else { + const tap_plan& plan = summary.plan(); + output << "bailed_out=false" + << ", plan=" << plan.first << ".." << plan.second; + if (plan == all_skipped_plan) { + output << ", all_skipped_reason=" << summary.all_skipped_reason(); + } else { + output << ", ok_count=" << summary.ok_count() + << ", not_ok_count=" << summary.not_ok_count(); + } + } + output << "}"; + return output; +} + + +/// Parses an input file containing the TAP output of a test program. +/// +/// \param filename Path to the file to parse. +/// +/// \return The parsed data in the form of a tap_summary. +/// +/// \throw load_error If there are any problems parsing the file. Such problems +/// should be considered as test program breakage. +engine::tap_summary +engine::parse_tap_output(const utils::fs::path& filename) +{ + std::ifstream input(filename.str().c_str()); + if (!input) + throw engine::load_error(filename, "Failed to open TAP output file"); + + try { + return tap_summary(tap_parser().parse(input)); + } catch (const engine::format_error& e) { + throw engine::load_error(filename, e.what()); + } catch (const text::error& e) { + throw engine::load_error(filename, e.what()); + } +} diff --git a/engine/tap_parser.hpp b/engine/tap_parser.hpp new file mode 100644 index 000000000000..84cea908f5ba --- /dev/null +++ b/engine/tap_parser.hpp @@ -0,0 +1,99 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/tap_parser.hpp +/// Utilities to parse TAP test program output. + +#if !defined(ENGINE_TAP_PARSER_HPP) +#define ENGINE_TAP_PARSER_HPP + +#include "engine/tap_parser_fwd.hpp" + +#include <cstddef> +#include <ostream> +#include <string> + +#include "utils/fs/path_fwd.hpp" + +namespace engine { + + +/// TAP plan representing all tests being skipped. +extern const engine::tap_plan all_skipped_plan; + + +/// TAP output representation and parser. +class tap_summary { + /// Whether the test program bailed out early or not. + bool _bailed_out; + + /// The TAP plan. Only valid if not bailed out. + tap_plan _plan; + + /// If not empty, the reason why all tests were skipped. + std::string _all_skipped_reason; + + /// Total number of 'ok' tests. Only valid if not balied out. + std::size_t _ok_count; + + /// Total number of 'not ok' tests. Only valid if not balied out. + std::size_t _not_ok_count; + + tap_summary(const bool, const tap_plan&, const std::string&, + const std::size_t, const std::size_t); + +public: + // Yes, these three constructors indicate that we really ought to have three + // different classes and select between them at runtime. But doing so would + // be overly complex for our really simple needs here. + static tap_summary new_bailed_out(void); + static tap_summary new_all_skipped(const std::string&); + static tap_summary new_results(const tap_plan&, const std::size_t, + const std::size_t); + + bool bailed_out(void) const; + const tap_plan& plan(void) const; + const std::string& all_skipped_reason(void) const; + std::size_t ok_count(void) const; + std::size_t not_ok_count(void) const; + + bool operator==(const tap_summary&) const; + bool operator!=(const tap_summary&) const; +}; + + +std::ostream& operator<<(std::ostream&, const tap_summary&); + + +tap_summary parse_tap_output(const utils::fs::path&); + + +} // namespace engine + + +#endif // !defined(ENGINE_TAP_PARSER_HPP) diff --git a/engine/tap_parser_fwd.hpp b/engine/tap_parser_fwd.hpp new file mode 100644 index 000000000000..481ed2f42267 --- /dev/null +++ b/engine/tap_parser_fwd.hpp @@ -0,0 +1,50 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/tap_parser_fwd.hpp +/// Forward declarations for engine/tap_parser.hpp + +#if !defined(ENGINE_TAP_PARSER_FWD_HPP) +#define ENGINE_TAP_PARSER_FWD_HPP + +#include <cstddef> +#include <utility> + +namespace engine { + + +/// Representation of the TAP plan line. +typedef std::pair< std::size_t, std::size_t > tap_plan; + + +class tap_summary; + + +} // namespace engine + +#endif // !defined(ENGINE_TAP_PARSER_FWD_HPP) diff --git a/engine/tap_parser_test.cpp b/engine/tap_parser_test.cpp new file mode 100644 index 000000000000..af993bfab4ab --- /dev/null +++ b/engine/tap_parser_test.cpp @@ -0,0 +1,465 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/tap_parser.hpp" + +#include <fstream> + +#include <atf-c++.hpp> + +#include "engine/exceptions.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" + +namespace fs = utils::fs; + + +namespace { + + +/// Helper to execute parse_tap_output() on inline text contents. +/// +/// \param contents The TAP output to parse. +/// +/// \return The tap_summary object resultingafter the parse. +/// +/// \throw engine::load_error If parse_tap_output() fails. +static engine::tap_summary +do_parse(const std::string& contents) +{ + std::ofstream output("tap.txt"); + ATF_REQUIRE(output); + output << contents; + output.close(); + return engine::parse_tap_output(fs::path("tap.txt")); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__bailed_out); +ATF_TEST_CASE_BODY(tap_summary__bailed_out) +{ + const engine::tap_summary summary = engine::tap_summary::new_bailed_out(); + ATF_REQUIRE(summary.bailed_out()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__some_results); +ATF_TEST_CASE_BODY(tap_summary__some_results) +{ + const engine::tap_summary summary = engine::tap_summary::new_results( + engine::tap_plan(1, 5), 3, 2); + ATF_REQUIRE(!summary.bailed_out()); + ATF_REQUIRE_EQ(engine::tap_plan(1, 5), summary.plan()); + ATF_REQUIRE_EQ(3, summary.ok_count()); + ATF_REQUIRE_EQ(2, summary.not_ok_count()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__all_skipped); +ATF_TEST_CASE_BODY(tap_summary__all_skipped) +{ + const engine::tap_summary summary = engine::tap_summary::new_all_skipped( + "Skipped"); + ATF_REQUIRE(!summary.bailed_out()); + ATF_REQUIRE_EQ(engine::tap_plan(1, 0), summary.plan()); + ATF_REQUIRE_EQ("Skipped", summary.all_skipped_reason()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__equality_operators); +ATF_TEST_CASE_BODY(tap_summary__equality_operators) +{ + const engine::tap_summary bailed_out = + engine::tap_summary::new_bailed_out(); + const engine::tap_summary all_skipped_1 = + engine::tap_summary::new_all_skipped("Reason 1"); + const engine::tap_summary results_1 = + engine::tap_summary::new_results(engine::tap_plan(1, 5), 3, 2); + + // Self-equality checks. + ATF_REQUIRE( bailed_out == bailed_out); + ATF_REQUIRE(!(bailed_out != bailed_out)); + ATF_REQUIRE( all_skipped_1 == all_skipped_1); + ATF_REQUIRE(!(all_skipped_1 != all_skipped_1)); + ATF_REQUIRE( results_1 == results_1); + ATF_REQUIRE(!(results_1 != results_1)); + + // Cross-equality checks. + ATF_REQUIRE(!(bailed_out == all_skipped_1)); + ATF_REQUIRE( bailed_out != all_skipped_1); + ATF_REQUIRE(!(bailed_out == results_1)); + ATF_REQUIRE( bailed_out != results_1); + ATF_REQUIRE(!(all_skipped_1 == results_1)); + ATF_REQUIRE( all_skipped_1 != results_1); + + // Checks for the all_skipped "type". + const engine::tap_summary all_skipped_2 = + engine::tap_summary::new_all_skipped("Reason 2"); + ATF_REQUIRE(!(all_skipped_1 == all_skipped_2)); + ATF_REQUIRE( all_skipped_1 != all_skipped_2); + + + // Checks for the results "type", different plan. + const engine::tap_summary results_2 = + engine::tap_summary::new_results(engine::tap_plan(2, 6), + results_1.ok_count(), + results_1.not_ok_count()); + ATF_REQUIRE(!(results_1 == results_2)); + ATF_REQUIRE( results_1 != results_2); + + + // Checks for the results "type", different counts. + const engine::tap_summary results_3 = + engine::tap_summary::new_results(results_1.plan(), + results_1.not_ok_count(), + results_1.ok_count()); + ATF_REQUIRE(!(results_1 == results_3)); + ATF_REQUIRE( results_1 != results_3); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__output); +ATF_TEST_CASE_BODY(tap_summary__output) +{ + { + const engine::tap_summary summary = + engine::tap_summary::new_bailed_out(); + ATF_REQUIRE_EQ( + "tap_summary{bailed_out=true}", + (F("%s") % summary).str()); + } + + { + const engine::tap_summary summary = + engine::tap_summary::new_results(engine::tap_plan(5, 10), 2, 4); + ATF_REQUIRE_EQ( + "tap_summary{bailed_out=false, plan=5..10, ok_count=2, " + "not_ok_count=4}", + (F("%s") % summary).str()); + } + + { + const engine::tap_summary summary = + engine::tap_summary::new_all_skipped("Who knows"); + ATF_REQUIRE_EQ( + "tap_summary{bailed_out=false, plan=1..0, " + "all_skipped_reason=Who knows}", + (F("%s") % summary).str()); + } +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__only_one_result); +ATF_TEST_CASE_BODY(parse_tap_output__only_one_result) +{ + const engine::tap_summary summary = do_parse( + "1..1\n" + "ok - 1\n"); + + const engine::tap_summary exp_summary = + engine::tap_summary::new_results(engine::tap_plan(1, 1), 1, 0); + ATF_REQUIRE_EQ(exp_summary, summary); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__all_pass); +ATF_TEST_CASE_BODY(parse_tap_output__all_pass) +{ + const engine::tap_summary summary = do_parse( + "1..8\n" + "ok - 1\n" + " Some diagnostic message\n" + "ok - 2 This test also passed\n" + "garbage line\n" + "ok - 3 This test passed\n" + "not ok 4 # SKIP Some reason\n" + "not ok 5 # TODO Another reason\n" + "ok - 6 Doesn't make a difference SKIP\n" + "ok - 7 Doesn't make a difference either TODO\n" + "ok # Also works without a number\n"); + + const engine::tap_summary exp_summary = + engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0); + ATF_REQUIRE_EQ(exp_summary, summary); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__some_fail); +ATF_TEST_CASE_BODY(parse_tap_output__some_fail) +{ + const engine::tap_summary summary = do_parse( + "garbage line\n" + "not ok - 1 This test failed\n" + "ok - 2 This test passed\n" + "not ok - 3 This test failed\n" + "1..6\n" + "not ok - 4 This test failed\n" + "ok - 5 This test passed\n" + "not ok # Fails as well without a number\n"); + + const engine::tap_summary exp_summary = + engine::tap_summary::new_results(engine::tap_plan(1, 6), 2, 4); + ATF_REQUIRE_EQ(exp_summary, summary); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_and_todo_variants); +ATF_TEST_CASE_BODY(parse_tap_output__skip_and_todo_variants) +{ + const engine::tap_summary summary = do_parse( + "1..8\n" + "not ok - 1 # SKIP Some reason\n" + "not ok - 2 # skip Some reason\n" + "not ok - 3 # Skipped Some reason\n" + "not ok - 4 # skipped Some reason\n" + "not ok - 5 # Skipped: Some reason\n" + "not ok - 6 # skipped: Some reason\n" + "not ok - 7 # TODO Some reason\n" + "not ok - 8 # todo Some reason\n"); + + const engine::tap_summary exp_summary = + engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0); + ATF_REQUIRE_EQ(exp_summary, summary); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_with_reason); +ATF_TEST_CASE_BODY(parse_tap_output__skip_all_with_reason) +{ + const engine::tap_summary summary = do_parse( + "1..0 SKIP Some reason for skipping\n" + "ok - 1\n" + " Some diagnostic message\n" + "ok - 6 Doesn't make a difference SKIP\n" + "ok - 7 Doesn't make a difference either TODO\n"); + + const engine::tap_summary exp_summary = + engine::tap_summary::new_all_skipped("Some reason for skipping"); + ATF_REQUIRE_EQ(exp_summary, summary); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_without_reason); +ATF_TEST_CASE_BODY(parse_tap_output__skip_all_without_reason) +{ + const engine::tap_summary summary = do_parse( + "1..0 unrecognized # garbage skip\n"); + + const engine::tap_summary exp_summary = + engine::tap_summary::new_all_skipped("No reason specified"); + ATF_REQUIRE_EQ(exp_summary, summary); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_invalid); +ATF_TEST_CASE_BODY(parse_tap_output__skip_all_invalid) +{ + ATF_REQUIRE_THROW_RE(engine::load_error, + "Skipped plan must be 1\\.\\.0", + do_parse("1..3 # skip\n")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__plan_at_end); +ATF_TEST_CASE_BODY(parse_tap_output__plan_at_end) +{ + const engine::tap_summary summary = do_parse( + "ok - 1\n" + " Some diagnostic message\n" + "ok - 2 This test also passed\n" + "garbage line\n" + "ok - 3 This test passed\n" + "not ok 4 # SKIP Some reason\n" + "not ok 5 # TODO Another reason\n" + "ok - 6 Doesn't make a difference SKIP\n" + "ok - 7 Doesn't make a difference either TODO\n" + "1..7\n"); + + const engine::tap_summary exp_summary = + engine::tap_summary::new_results(engine::tap_plan(1, 7), 7, 0); + ATF_REQUIRE_EQ(exp_summary, summary); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__stray_oks); +ATF_TEST_CASE_BODY(parse_tap_output__stray_oks) +{ + const engine::tap_summary summary = do_parse( + "1..3\n" + "ok - 1\n" + "ok\n" + "ok - 2 This test also passed\n" + "not ok\n" + "ok - 3 This test passed\n"); + + const engine::tap_summary exp_summary = + engine::tap_summary::new_results(engine::tap_plan(1, 3), 3, 0); + ATF_REQUIRE_EQ(exp_summary, summary); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__no_plan); +ATF_TEST_CASE_BODY(parse_tap_output__no_plan) +{ + ATF_REQUIRE_THROW_RE( + engine::load_error, + "Output did not contain any TAP plan", + do_parse( + "not ok - 1 This test failed\n" + "ok - 2 This test passed\n")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__double_plan); +ATF_TEST_CASE_BODY(parse_tap_output__double_plan) +{ + ATF_REQUIRE_THROW_RE( + engine::load_error, + "Found duplicate plan", + do_parse( + "garbage line\n" + "1..5\n" + "not ok - 1 This test failed\n" + "ok - 2 This test passed\n" + "1..8\n" + "ok\n")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_plan); +ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_plan) +{ + ATF_REQUIRE_THROW_RE( + engine::load_error, + "Reported plan differs from actual executed tests", + do_parse( + "1..3\n" + "not ok - 1 This test failed\n" + "ok - 2 This test passed\n")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_trailing_plan); +ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_trailing_plan) +{ + ATF_REQUIRE_THROW_RE( + engine::load_error, + "Reported plan differs from actual executed tests", + do_parse( + "not ok - 1 This test failed\n" + "ok - 2 This test passed\n" + "1..3\n")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__insane_plan); +ATF_TEST_CASE_BODY(parse_tap_output__insane_plan) +{ + ATF_REQUIRE_THROW_RE( + engine::load_error, "Invalid value", + do_parse("120830981209831..234891793874080981092803981092312\n")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__reversed_plan); +ATF_TEST_CASE_BODY(parse_tap_output__reversed_plan) +{ + ATF_REQUIRE_THROW_RE(engine::load_error, + "Found reversed plan 8\\.\\.5", + do_parse("8..5\n")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out); +ATF_TEST_CASE_BODY(parse_tap_output__bail_out) +{ + const engine::tap_summary summary = do_parse( + "1..3\n" + "not ok - 1 This test failed\n" + "Bail out! There is some unknown problem\n" + "ok - 2 This test passed\n"); + + const engine::tap_summary exp_summary = + engine::tap_summary::new_bailed_out(); + ATF_REQUIRE_EQ(exp_summary, summary); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out_wins_over_no_plan); +ATF_TEST_CASE_BODY(parse_tap_output__bail_out_wins_over_no_plan) +{ + const engine::tap_summary summary = do_parse( + "not ok - 1 This test failed\n" + "Bail out! There is some unknown problem\n" + "ok - 2 This test passed\n"); + + const engine::tap_summary exp_summary = + engine::tap_summary::new_bailed_out(); + ATF_REQUIRE_EQ(exp_summary, summary); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__open_failure); +ATF_TEST_CASE_BODY(parse_tap_output__open_failure) +{ + ATF_REQUIRE_THROW_RE(engine::load_error, "hello.txt.*Failed to open", + engine::parse_tap_output(fs::path("hello.txt"))); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, tap_summary__bailed_out); + ATF_ADD_TEST_CASE(tcs, tap_summary__some_results); + ATF_ADD_TEST_CASE(tcs, tap_summary__all_skipped); + ATF_ADD_TEST_CASE(tcs, tap_summary__equality_operators); + ATF_ADD_TEST_CASE(tcs, tap_summary__output); + + ATF_ADD_TEST_CASE(tcs, parse_tap_output__only_one_result); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__all_pass); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__some_fail); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_and_todo_variants); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_without_reason); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_with_reason); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_invalid); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__plan_at_end); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__stray_oks); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__no_plan); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__double_plan); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_plan); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_trailing_plan); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__insane_plan); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__reversed_plan); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out_wins_over_no_plan); + ATF_ADD_TEST_CASE(tcs, parse_tap_output__open_failure); +} diff --git a/engine/tap_test.cpp b/engine/tap_test.cpp new file mode 100644 index 000000000000..f4253a68e727 --- /dev/null +++ b/engine/tap_test.cpp @@ -0,0 +1,218 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/tap.hpp" + +extern "C" { +#include <signal.h> +} + +#include <atf-c++.hpp> + +#include "engine/config.hpp" +#include "engine/scheduler.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/env.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace scheduler = engine::scheduler; + +using utils::none; + + +namespace { + + +/// Copies the tap helper to the work directory, selecting a specific helper. +/// +/// \param tc Pointer to the calling test case, to obtain srcdir. +/// \param name Name of the new binary to create. Must match the name of a +/// valid helper, as the binary name is used to select it. +static void +copy_tap_helper(const atf::tests::tc* tc, const char* name) +{ + const fs::path srcdir(tc->get_config_var("srcdir")); + atf::utils::copy_file((srcdir / "tap_helpers").str(), name); +} + + +/// Runs one tap test program and checks its result. +/// +/// \param tc Pointer to the calling test case, to obtain srcdir. +/// \param test_case_name Name of the "test case" to select from the helper +/// program. +/// \param exp_result The expected result. +/// \param metadata The test case metadata. +/// \param user_config User-provided configuration variables. +static void +run_one(const atf::tests::tc* tc, const char* test_case_name, + const model::test_result& exp_result, + const model::metadata& metadata = model::metadata_builder().build(), + const config::tree& user_config = engine::empty_config()) +{ + copy_tap_helper(tc, test_case_name); + const model::test_program_ptr program = model::test_program_builder( + "tap", fs::path(test_case_name), fs::current_path(), "the-suite") + .add_test_case("main", metadata).build_ptr(); + + scheduler::scheduler_handle handle = scheduler::setup(); + (void)handle.spawn_test(program, "main", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: "); + atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: "); + ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(list); +ATF_TEST_CASE_BODY(list) +{ + const model::test_program program = model::test_program_builder( + "tap", fs::path("non-existent"), fs::path("."), "unused-suite") + .build(); + + scheduler::scheduler_handle handle = scheduler::setup(); + const model::test_cases_map test_cases = handle.list_tests( + &program, engine::empty_config()); + handle.cleanup(); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("main").build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__all_tests_pass); +ATF_TEST_CASE_BODY(test__all_tests_pass) +{ + const model::test_result exp_result(model::test_result_passed); + run_one(this, "pass", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__some_tests_fail); +ATF_TEST_CASE_BODY(test__some_tests_fail) +{ + const model::test_result exp_result(model::test_result_failed, + "2 of 5 tests failed"); + run_one(this, "fail", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__all_tests_pass_but_exit_failure); +ATF_TEST_CASE_BODY(test__all_tests_pass_but_exit_failure) +{ + const model::test_result exp_result( + model::test_result_broken, + "Dubious test program: reported all tests as passed but returned exit " + "code 70"); + run_one(this, "pass_but_exit_failure", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__signal_is_broken); +ATF_TEST_CASE_BODY(test__signal_is_broken) +{ + const model::test_result exp_result(model::test_result_broken, + F("Received signal %s") % SIGABRT); + run_one(this, "crash", exp_result); +} + + +ATF_TEST_CASE(test__timeout_is_broken); +ATF_TEST_CASE_HEAD(test__timeout_is_broken) +{ + set_md_var("timeout", "60"); +} +ATF_TEST_CASE_BODY(test__timeout_is_broken) +{ + utils::setenv("CONTROL_DIR", fs::current_path().str()); + + const model::metadata metadata = model::metadata_builder() + .set_timeout(datetime::delta(1, 0)).build(); + const model::test_result exp_result(model::test_result_broken, + "Test case timed out"); + run_one(this, "timeout", exp_result, metadata); + + ATF_REQUIRE(!atf::utils::file_exists("cookie")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__configuration_variables); +ATF_TEST_CASE_BODY(test__configuration_variables) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.a-suite.first", "unused"); + user_config.set_string("test_suites.the-suite.first", "some value"); + user_config.set_string("test_suites.the-suite.second", "some other value"); + user_config.set_string("test_suites.other-suite.first", "unused"); + + const model::test_result exp_result(model::test_result_passed); + run_one(this, "check_configuration_variables", exp_result, + model::metadata_builder().build(), user_config); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + scheduler::register_interface( + "tap", std::shared_ptr< scheduler::interface >( + new engine::tap_interface())); + + ATF_ADD_TEST_CASE(tcs, list); + + ATF_ADD_TEST_CASE(tcs, test__all_tests_pass); + ATF_ADD_TEST_CASE(tcs, test__all_tests_pass_but_exit_failure); + ATF_ADD_TEST_CASE(tcs, test__some_tests_fail); + ATF_ADD_TEST_CASE(tcs, test__signal_is_broken); + ATF_ADD_TEST_CASE(tcs, test__timeout_is_broken); + ATF_ADD_TEST_CASE(tcs, test__configuration_variables); +} |