aboutsummaryrefslogtreecommitdiffstats
path: root/engine/scheduler_test.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'engine/scheduler_test.cpp')
-rw-r--r--engine/scheduler_test.cpp1242
1 files changed, 1242 insertions, 0 deletions
diff --git a/engine/scheduler_test.cpp b/engine/scheduler_test.cpp
new file mode 100644
index 000000000000..e144761d8f01
--- /dev/null
+++ b/engine/scheduler_test.cpp
@@ -0,0 +1,1242 @@
+// Copyright 2014 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "engine/scheduler.hpp"
+
+extern "C" {
+#include <sys/types.h>
+
+#include <signal.h>
+#include <unistd.h>
+}
+
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <string>
+
+#include <atf-c++.hpp>
+
+#include "engine/config.hpp"
+#include "engine/exceptions.hpp"
+#include "model/context.hpp"
+#include "model/metadata.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "model/test_result.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/datetime.hpp"
+#include "utils/defs.hpp"
+#include "utils/env.hpp"
+#include "utils/format/containers.ipp"
+#include "utils/format/macros.hpp"
+#include "utils/fs/operations.hpp"
+#include "utils/fs/path.hpp"
+#include "utils/optional.ipp"
+#include "utils/passwd.hpp"
+#include "utils/process/status.hpp"
+#include "utils/sanity.hpp"
+#include "utils/stacktrace.hpp"
+#include "utils/stream.hpp"
+#include "utils/test_utils.ipp"
+#include "utils/text/exceptions.hpp"
+#include "utils/text/operations.ipp"
+
+namespace config = utils::config;
+namespace datetime = utils::datetime;
+namespace fs = utils::fs;
+namespace passwd = utils::passwd;
+namespace process = utils::process;
+namespace scheduler = engine::scheduler;
+namespace text = utils::text;
+
+using utils::none;
+using utils::optional;
+
+
+namespace {
+
+
+/// Checks if a string starts with a prefix.
+///
+/// \param str The string to be tested.
+/// \param prefix The prefix to look for.
+///
+/// \return True if the string is prefixed as specified.
+static bool
+starts_with(const std::string& str, const std::string& prefix)
+{
+ return (str.length() >= prefix.length() &&
+ str.substr(0, prefix.length()) == prefix);
+}
+
+
+/// Strips a prefix from a string and converts the rest to an integer.
+///
+/// \param str The string to be tested.
+/// \param prefix The prefix to strip from the string.
+///
+/// \return The part of the string after the prefix converted to an integer.
+static int
+suffix_to_int(const std::string& str, const std::string& prefix)
+{
+ PRE(starts_with(str, prefix));
+ try {
+ return text::to_type< int >(str.substr(prefix.length()));
+ } catch (const text::value_error& error) {
+ std::cerr << F("Failed: %s\n") % error.what();
+ std::abort();
+ }
+}
+
+
+/// Mock interface definition for testing.
+///
+/// This scheduler interface does not execute external binaries. It is designed
+/// to simulate the scheduler of various programs with different exit statuses.
+class mock_interface : public scheduler::interface {
+ /// Executes the subprocess simulating an exec.
+ ///
+ /// This is just a simple wrapper over _exit(2) because we cannot use
+ /// std::exit on exit from this mock interface. The reason is that we do
+ /// not want to invoke any destructors as otherwise we'd clear up the global
+ /// scheduler state by mistake. This wouldn't be a major problem if it
+ /// wasn't because doing so deletes on-disk files and we want to leave them
+ /// in place so that the parent process can test for them!
+ ///
+ /// \param exit_code Exit code.
+ void
+ do_exit(const int exit_code) const UTILS_NORETURN
+ {
+ std::cout.flush();
+ std::cerr.flush();
+ ::_exit(exit_code);
+ }
+
+ /// Executes a test case that creates various files and then fails.
+ void
+ exec_create_files_and_fail(void) const UTILS_NORETURN
+ {
+ std::cerr << "This should not be clobbered\n";
+ atf::utils::create_file("first file", "");
+ atf::utils::create_file("second-file", "");
+ fs::mkdir_p(fs::path("dir1/dir2"), 0755);
+ ::kill(::getpid(), SIGTERM);
+ std::abort();
+ }
+
+ /// Executes a test case that deletes all files in the current directory.
+ ///
+ /// This is intended to validate that the test runs in an empty directory,
+ /// separate from any control files that the scheduler may have created.
+ void
+ exec_delete_all(void) const UTILS_NORETURN
+ {
+ const int exit_code = ::system("rm *") == -1
+ ? EXIT_FAILURE : EXIT_SUCCESS;
+
+ // Recreate our own cookie.
+ atf::utils::create_file("exec_test_was_called", "");
+
+ do_exit(exit_code);
+ }
+
+ /// Executes a test case that returns a specific exit code.
+ ///
+ /// \param exit_code Exit status to terminate the program with.
+ void
+ exec_exit(const int exit_code) const UTILS_NORETURN
+ {
+ do_exit(exit_code);
+ }
+
+ /// Executes a test case that just fails.
+ void
+ exec_fail(void) const UTILS_NORETURN
+ {
+ std::cerr << "This should not be clobbered\n";
+ ::kill(::getpid(), SIGTERM);
+ std::abort();
+ }
+
+ /// Executes a test case that prints all input parameters to the functor.
+ ///
+ /// \param test_program The test program to execute.
+ /// \param test_case_name Name of the test case to invoke, which must be a
+ /// number.
+ /// \param vars User-provided variables to pass to the test program.
+ void
+ exec_print_params(const model::test_program& test_program,
+ const std::string& test_case_name,
+ const config::properties_map& vars) const
+ UTILS_NORETURN
+ {
+ std::cout << F("Test program: %s\n") % test_program.relative_path();
+ std::cout << F("Test case: %s\n") % test_case_name;
+ for (config::properties_map::const_iterator iter = vars.begin();
+ iter != vars.end(); ++iter) {
+ std::cout << F("%s=%s\n") % (*iter).first % (*iter).second;
+ }
+
+ std::cerr << F("stderr: %s\n") % test_case_name;
+
+ do_exit(EXIT_SUCCESS);
+ }
+
+public:
+ /// Executes a test program's list operation.
+ ///
+ /// This method is intended to be called within a subprocess and is expected
+ /// to terminate execution either by exec(2)ing the test program or by
+ /// exiting with a failure.
+ ///
+ /// \param test_program The test program to execute.
+ /// \param vars User-provided variables to pass to the test program.
+ void
+ exec_list(const model::test_program& test_program,
+ const config::properties_map& vars)
+ const UTILS_NORETURN
+ {
+ const std::string name = test_program.absolute_path().leaf_name();
+
+ std::cerr << name;
+ std::cerr.flush();
+ if (name == "check_i_exist") {
+ if (fs::exists(test_program.absolute_path())) {
+ std::cout << "found\n";
+ do_exit(EXIT_SUCCESS);
+ } else {
+ std::cout << "not_found\n";
+ do_exit(EXIT_FAILURE);
+ }
+ } else if (name == "empty") {
+ do_exit(EXIT_SUCCESS);
+ } else if (name == "misbehave") {
+ utils::abort_without_coredump();
+ } else if (name == "timeout") {
+ std::cout << "sleeping\n";
+ std::cout.flush();
+ ::sleep(100);
+ utils::abort_without_coredump();
+ } else if (name == "vars") {
+ for (config::properties_map::const_iterator iter = vars.begin();
+ iter != vars.end(); ++iter) {
+ std::cout << F("%s_%s\n") % (*iter).first % (*iter).second;
+ }
+ do_exit(15);
+ } else {
+ std::abort();
+ }
+ }
+
+ /// Computes the test cases list of a test program.
+ ///
+ /// \param status The termination status of the subprocess used to execute
+ /// the exec_test() method or none if the test timed out.
+ /// \param stdout_path Path to the file containing the stdout of the test.
+ /// \param stderr_path Path to the file containing the stderr of the test.
+ ///
+ /// \return A list of test cases.
+ model::test_cases_map
+ parse_list(const optional< process::status >& status,
+ const fs::path& stdout_path,
+ const fs::path& stderr_path) const
+ {
+ const std::string name = utils::read_file(stderr_path);
+ if (name == "check_i_exist") {
+ ATF_REQUIRE(status.get().exited());
+ ATF_REQUIRE_EQ(EXIT_SUCCESS, status.get().exitstatus());
+ } else if (name == "empty") {
+ ATF_REQUIRE(status.get().exited());
+ ATF_REQUIRE_EQ(EXIT_SUCCESS, status.get().exitstatus());
+ } else if (name == "misbehave") {
+ throw std::runtime_error("misbehaved in parse_list");
+ } else if (name == "timeout") {
+ ATF_REQUIRE(!status);
+ } else if (name == "vars") {
+ ATF_REQUIRE(status.get().exited());
+ ATF_REQUIRE_EQ(15, status.get().exitstatus());
+ } else {
+ ATF_FAIL("Invalid stderr contents; got " + name);
+ }
+
+ model::test_cases_map_builder test_cases_builder;
+
+ std::ifstream input(stdout_path.c_str());
+ ATF_REQUIRE(input);
+ std::string line;
+ while (std::getline(input, line).good()) {
+ test_cases_builder.add(line);
+ }
+
+ return test_cases_builder.build();
+ }
+
+ /// Executes a test case of the test program.
+ ///
+ /// This method is intended to be called within a subprocess and is expected
+ /// to terminate execution either by exec(2)ing the test program or by
+ /// exiting with a failure.
+ ///
+ /// \param test_program The test program to execute.
+ /// \param test_case_name Name of the test case to invoke.
+ /// \param vars User-provided variables to pass to the test program.
+ /// \param control_directory Directory where the interface may place control
+ /// files.
+ void
+ exec_test(const model::test_program& test_program,
+ const std::string& test_case_name,
+ const config::properties_map& vars,
+ const fs::path& control_directory) const
+ {
+ const fs::path cookie = control_directory / "exec_test_was_called";
+ std::ofstream control_file(cookie.c_str());
+ if (!control_file) {
+ std::cerr << "Failed to create " << cookie << '\n';
+ std::abort();
+ }
+ control_file << test_case_name;
+ control_file.close();
+
+ if (test_case_name == "check_i_exist") {
+ do_exit(fs::exists(test_program.absolute_path()) ? 0 : 1);
+ } else if (starts_with(test_case_name, "cleanup_timeout")) {
+ exec_exit(EXIT_SUCCESS);
+ } else if (starts_with(test_case_name, "create_files_and_fail")) {
+ exec_create_files_and_fail();
+ } else if (test_case_name == "delete_all") {
+ exec_delete_all();
+ } else if (starts_with(test_case_name, "exit ")) {
+ exec_exit(suffix_to_int(test_case_name, "exit "));
+ } else if (starts_with(test_case_name, "fail")) {
+ exec_fail();
+ } else if (starts_with(test_case_name, "fail_body_fail_cleanup")) {
+ exec_fail();
+ } else if (starts_with(test_case_name, "fail_body_pass_cleanup")) {
+ exec_fail();
+ } else if (starts_with(test_case_name, "pass_body_fail_cleanup")) {
+ exec_exit(EXIT_SUCCESS);
+ } else if (starts_with(test_case_name, "print_params")) {
+ exec_print_params(test_program, test_case_name, vars);
+ } else if (starts_with(test_case_name, "skip_body_pass_cleanup")) {
+ exec_exit(EXIT_SUCCESS);
+ } else {
+ std::cerr << "Unknown test case " << test_case_name << '\n';
+ std::abort();
+ }
+ }
+
+ /// Executes a test cleanup routine of the test program.
+ ///
+ /// This method is intended to be called within a subprocess and is expected
+ /// to terminate execution either by exec(2)ing the test program or by
+ /// exiting with a failure.
+ ///
+ /// \param test_case_name Name of the test case to invoke.
+ void
+ exec_cleanup(const model::test_program& /* test_program */,
+ const std::string& test_case_name,
+ const config::properties_map& /* vars */,
+ const fs::path& /* control_directory */) const
+ {
+ std::cout << "exec_cleanup was called\n";
+ std::cout.flush();
+
+ if (starts_with(test_case_name, "cleanup_timeout")) {
+ ::sleep(100);
+ std::abort();
+ } else if (starts_with(test_case_name, "fail_body_fail_cleanup")) {
+ exec_fail();
+ } else if (starts_with(test_case_name, "fail_body_pass_cleanup")) {
+ exec_exit(EXIT_SUCCESS);
+ } else if (starts_with(test_case_name, "pass_body_fail_cleanup")) {
+ exec_fail();
+ } else if (starts_with(test_case_name, "skip_body_pass_cleanup")) {
+ exec_exit(EXIT_SUCCESS);
+ } else {
+ std::cerr << "Should not have been called for a test without "
+ "a cleanup routine" << '\n';
+ std::abort();
+ }
+ }
+
+ /// Computes the result of a test case based on its termination status.
+ ///
+ /// \param status The termination status of the subprocess used to execute
+ /// the exec_test() method or none if the test timed out.
+ /// \param control_directory Path to the directory where the interface may
+ /// have placed control files.
+ /// \param stdout_path Path to the file containing the stdout of the test.
+ /// \param stderr_path Path to the file containing the stderr of the test.
+ ///
+ /// \return A test result.
+ model::test_result
+ compute_result(const optional< process::status >& status,
+ const fs::path& control_directory,
+ const fs::path& stdout_path,
+ const fs::path& stderr_path) const
+ {
+ // Do not use any ATF_* macros here. Some of the tests below invoke
+ // this code in a subprocess, and terminating such subprocess due to a
+ // failed ATF_* macro yields mysterious failures that are incredibly
+ // hard to debug. (Case in point: the signal_handling test is racy by
+ // nature, and the test run by exec_test() above may not have created
+ // the cookie we expect below. We don't want to "silently" exit if the
+ // file is not there.)
+
+ if (!status) {
+ return model::test_result(model::test_result_broken,
+ "Timed out");
+ }
+
+ if (status.get().exited()) {
+ // Only sanity-check the work directory-related parameters in case
+ // of a clean exit. In all other cases, there is no guarantee that
+ // these were ever created.
+ const fs::path cookie = control_directory / "exec_test_was_called";
+ if (!atf::utils::file_exists(cookie.str())) {
+ return model::test_result(
+ model::test_result_broken,
+ "compute_result's control_directory does not seem to point "
+ "to the right location");
+ }
+ const std::string test_case_name = utils::read_file(cookie);
+
+ if (!atf::utils::file_exists(stdout_path.str())) {
+ return model::test_result(
+ model::test_result_broken,
+ "compute_result's stdout_path does not exist");
+ }
+ if (!atf::utils::file_exists(stderr_path.str())) {
+ return model::test_result(
+ model::test_result_broken,
+ "compute_result's stderr_path does not exist");
+ }
+
+ if (test_case_name == "skip_body_pass_cleanup") {
+ return model::test_result(
+ model::test_result_skipped,
+ F("Exit %s") % status.get().exitstatus());
+ } else {
+ return model::test_result(
+ model::test_result_passed,
+ F("Exit %s") % status.get().exitstatus());
+ }
+ } else {
+ return model::test_result(
+ model::test_result_failed,
+ F("Signal %s") % status.get().termsig());
+ }
+ }
+};
+
+
+} // anonymous namespace
+
+
+/// Runs list_tests on the scheduler and returns the results.
+///
+/// \param test_name The name of the test supported by our exec_list function.
+/// \param user_config Optional user settings for the test.
+///
+/// \return The loaded list of test cases.
+static model::test_cases_map
+check_integration_list(const char* test_name, const fs::path root,
+ const config::tree& user_config = engine::empty_config())
+{
+ const model::test_program program = model::test_program_builder(
+ "mock", fs::path(test_name), root, "the-suite")
+ .build();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+ const model::test_cases_map test_cases = handle.list_tests(&program,
+ user_config);
+ handle.cleanup();
+
+ return test_cases;
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_some);
+ATF_TEST_CASE_BODY(integration__list_some)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.first", "test");
+ user_config.set_string("test_suites.the-suite.second", "TEST");
+ user_config.set_string("test_suites.abc.unused", "unused");
+
+ const model::test_cases_map test_cases = check_integration_list(
+ "vars", fs::path("."), user_config);
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("first_test").add("second_TEST").build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_check_paths);
+ATF_TEST_CASE_BODY(integration__list_check_paths)
+{
+ fs::mkdir_p(fs::path("dir1/dir2/dir3"), 0755);
+ atf::utils::create_file("dir1/dir2/dir3/check_i_exist", "");
+
+ const model::test_cases_map test_cases = check_integration_list(
+ "dir2/dir3/check_i_exist", fs::path("dir1"));
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("found").build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_timeout);
+ATF_TEST_CASE_BODY(integration__list_timeout)
+{
+ scheduler::list_timeout = datetime::delta(1, 0);
+ const model::test_cases_map test_cases = check_integration_list(
+ "timeout", fs::path("."));
+
+ const model::test_cases_map exp_test_cases = model::test_cases_map_builder()
+ .add("sleeping").build();
+ ATF_REQUIRE_EQ(exp_test_cases, test_cases);
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_fail);
+ATF_TEST_CASE_BODY(integration__list_fail)
+{
+ const model::test_cases_map test_cases = check_integration_list(
+ "misbehave", fs::path("."));
+
+ ATF_REQUIRE_EQ(1, test_cases.size());
+ const model::test_case& test_case = test_cases.begin()->second;
+ ATF_REQUIRE_EQ("__test_cases_list__", test_case.name());
+ ATF_REQUIRE(test_case.fake_result());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_broken,
+ "misbehaved in parse_list"),
+ test_case.fake_result().get());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_empty);
+ATF_TEST_CASE_BODY(integration__list_empty)
+{
+ const model::test_cases_map test_cases = check_integration_list(
+ "empty", fs::path("."));
+
+ ATF_REQUIRE_EQ(1, test_cases.size());
+ const model::test_case& test_case = test_cases.begin()->second;
+ ATF_REQUIRE_EQ("__test_cases_list__", test_case.name());
+ ATF_REQUIRE(test_case.fake_result());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_broken,
+ "Empty test cases list"),
+ test_case.fake_result().get());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__run_one);
+ATF_TEST_CASE_BODY(integration__run_one)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("exit 41").build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ const scheduler::exec_handle exec_handle = handle.spawn_test(
+ program, "exit 41", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(exec_handle, result_handle->original_pid());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 41"),
+ test_result_handle->test_result());
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__run_many);
+ATF_TEST_CASE_BODY(integration__run_many)
+{
+ static const std::size_t num_test_programs = 30;
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ // We mess around with the "current time" below, so make sure the tests do
+ // not spuriously exceed their deadline by bumping it to a large number.
+ const model::metadata infinite_timeout = model::metadata_builder()
+ .set_timeout(datetime::delta(1000000L, 0)).build();
+
+ std::size_t total_tests = 0;
+ std::map< scheduler::exec_handle, model::test_program_ptr >
+ exp_test_programs;
+ std::map< scheduler::exec_handle, std::string > exp_test_case_names;
+ std::map< scheduler::exec_handle, datetime::timestamp > exp_start_times;
+ std::map< scheduler::exec_handle, int > exp_exit_statuses;
+ for (std::size_t i = 0; i < num_test_programs; ++i) {
+ const std::string test_case_0 = F("exit %s") % (i * 3 + 0);
+ const std::string test_case_1 = F("exit %s") % (i * 3 + 1);
+ const std::string test_case_2 = F("exit %s") % (i * 3 + 2);
+
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path(F("program-%s") % i),
+ fs::current_path(), "the-suite")
+ .set_metadata(infinite_timeout)
+ .add_test_case(test_case_0)
+ .add_test_case(test_case_1)
+ .add_test_case(test_case_2)
+ .build_ptr();
+
+ const datetime::timestamp start_time = datetime::timestamp::from_values(
+ 2014, 12, 8, 9, 40, 0, i);
+
+ scheduler::exec_handle exec_handle;
+
+ datetime::set_mock_now(start_time);
+ exec_handle = handle.spawn_test(program, test_case_0, user_config);
+ exp_test_programs.insert(std::make_pair(exec_handle, program));
+ exp_test_case_names.insert(std::make_pair(exec_handle, test_case_0));
+ exp_start_times.insert(std::make_pair(exec_handle, start_time));
+ exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3));
+ ++total_tests;
+
+ datetime::set_mock_now(start_time);
+ exec_handle = handle.spawn_test(program, test_case_1, user_config);
+ exp_test_programs.insert(std::make_pair(exec_handle, program));
+ exp_test_case_names.insert(std::make_pair(exec_handle, test_case_1));
+ exp_start_times.insert(std::make_pair(exec_handle, start_time));
+ exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3 + 1));
+ ++total_tests;
+
+ datetime::set_mock_now(start_time);
+ exec_handle = handle.spawn_test(program, test_case_2, user_config);
+ exp_test_programs.insert(std::make_pair(exec_handle, program));
+ exp_test_case_names.insert(std::make_pair(exec_handle, test_case_2));
+ exp_start_times.insert(std::make_pair(exec_handle, start_time));
+ exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3 + 2));
+ ++total_tests;
+ }
+
+ for (std::size_t i = 0; i < total_tests; ++i) {
+ const datetime::timestamp end_time = datetime::timestamp::from_values(
+ 2014, 12, 8, 9, 50, 10, i);
+ datetime::set_mock_now(end_time);
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+
+ const scheduler::exec_handle exec_handle =
+ result_handle->original_pid();
+
+ const model::test_program_ptr test_program = exp_test_programs.find(
+ exec_handle)->second;
+ const std::string& test_case_name = exp_test_case_names.find(
+ exec_handle)->second;
+ const datetime::timestamp& start_time = exp_start_times.find(
+ exec_handle)->second;
+ const int exit_status = exp_exit_statuses.find(exec_handle)->second;
+
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed,
+ F("Exit %s") % exit_status),
+ test_result_handle->test_result());
+
+ ATF_REQUIRE_EQ(test_program, test_result_handle->test_program());
+ ATF_REQUIRE_EQ(test_case_name, test_result_handle->test_case_name());
+
+ ATF_REQUIRE_EQ(start_time, result_handle->start_time());
+ ATF_REQUIRE_EQ(end_time, result_handle->end_time());
+
+ result_handle->cleanup();
+
+ ATF_REQUIRE(!atf::utils::file_exists(
+ result_handle->stdout_file().str()));
+ ATF_REQUIRE(!atf::utils::file_exists(
+ result_handle->stderr_file().str()));
+ ATF_REQUIRE(!atf::utils::file_exists(
+ result_handle->work_directory().str()));
+
+ result_handle.reset();
+ }
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__run_check_paths);
+ATF_TEST_CASE_BODY(integration__run_check_paths)
+{
+ fs::mkdir_p(fs::path("dir1/dir2/dir3"), 0755);
+ atf::utils::create_file("dir1/dir2/dir3/program", "");
+
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("dir2/dir3/program"), fs::path("dir1"), "the-suite")
+ .add_test_case("check_i_exist").build_ptr();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "check_i_exist", engine::default_config());
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
+ test_result_handle->test_result());
+
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__parameters_and_output);
+ATF_TEST_CASE_BODY(integration__parameters_and_output)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("print_params").build_ptr();
+
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.one", "first variable");
+ user_config.set_string("test_suites.the-suite.two", "second variable");
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ const scheduler::exec_handle exec_handle = handle.spawn_test(
+ program, "print_params", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+
+ ATF_REQUIRE_EQ(exec_handle, result_handle->original_pid());
+ ATF_REQUIRE_EQ(program, test_result_handle->test_program());
+ ATF_REQUIRE_EQ("print_params", test_result_handle->test_case_name());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
+ test_result_handle->test_result());
+
+ const fs::path stdout_file = result_handle->stdout_file();
+ ATF_REQUIRE(atf::utils::compare_file(
+ stdout_file.str(),
+ "Test program: the-program\n"
+ "Test case: print_params\n"
+ "one=first variable\n"
+ "two=second variable\n"));
+ const fs::path stderr_file = result_handle->stderr_file();
+ ATF_REQUIRE(atf::utils::compare_file(
+ stderr_file.str(), "stderr: print_params\n"));
+
+ result_handle->cleanup();
+ ATF_REQUIRE(!fs::exists(stdout_file));
+ ATF_REQUIRE(!fs::exists(stderr_file));
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__fake_result);
+ATF_TEST_CASE_BODY(integration__fake_result)
+{
+ const model::test_result fake_result(model::test_result_skipped,
+ "Some fake details");
+
+ model::test_cases_map test_cases;
+ test_cases.insert(model::test_cases_map::value_type(
+ "__fake__", model::test_case("__fake__", "ABC", fake_result)));
+
+ const model::test_program_ptr program(new model::test_program(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite",
+ model::metadata_builder().build(), test_cases));
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "__fake__", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(fake_result, test_result_handle->test_result());
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__head_skips);
+ATF_TEST_CASE_BODY(integration__cleanup__head_skips)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("skip_me",
+ model::metadata_builder()
+ .add_required_config("variable-that-does-not-exist")
+ .set_has_cleanup(true)
+ .build())
+ .build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "skip_me", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(model::test_result(
+ model::test_result_skipped,
+ "Required configuration property "
+ "'variable-that-does-not-exist' not defined"),
+ test_result_handle->test_result());
+ ATF_REQUIRE(!atf::utils::grep_file("exec_cleanup was called",
+ result_handle->stdout_file().str()));
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+/// Runs a test to verify the behavior of cleanup routines.
+///
+/// \param test_case The name of the test case to invoke.
+/// \param exp_result The expected test result of the execution.
+static void
+do_cleanup_test(const char* test_case,
+ const model::test_result& exp_result)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case(test_case)
+ .set_metadata(model::metadata_builder().set_has_cleanup(true).build())
+ .build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, test_case, user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result());
+ ATF_REQUIRE(atf::utils::compare_file(
+ result_handle->stdout_file().str(),
+ "exec_cleanup was called\n"));
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_skips);
+ATF_TEST_CASE_BODY(integration__cleanup__body_skips)
+{
+ do_cleanup_test(
+ "skip_body_pass_cleanup",
+ model::test_result(model::test_result_skipped, "Exit 0"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_bad__cleanup_ok);
+ATF_TEST_CASE_BODY(integration__cleanup__body_bad__cleanup_ok)
+{
+ do_cleanup_test(
+ "fail_body_pass_cleanup",
+ model::test_result(model::test_result_failed, "Signal 15"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_ok__cleanup_bad);
+ATF_TEST_CASE_BODY(integration__cleanup__body_ok__cleanup_bad)
+{
+ do_cleanup_test(
+ "pass_body_fail_cleanup",
+ model::test_result(model::test_result_broken, "Test case cleanup "
+ "did not terminate successfully"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_bad__cleanup_bad);
+ATF_TEST_CASE_BODY(integration__cleanup__body_bad__cleanup_bad)
+{
+ do_cleanup_test(
+ "fail_body_fail_cleanup",
+ model::test_result(model::test_result_failed, "Signal 15"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__timeout);
+ATF_TEST_CASE_BODY(integration__cleanup__timeout)
+{
+ scheduler::cleanup_timeout = datetime::delta(1, 0);
+ do_cleanup_test(
+ "cleanup_timeout",
+ model::test_result(model::test_result_broken, "Test case cleanup "
+ "timed out"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__check_requirements);
+ATF_TEST_CASE_BODY(integration__check_requirements)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("exit 12")
+ .set_metadata(model::metadata_builder()
+ .add_required_config("abcde").build())
+ .build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "exit 12", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(model::test_result(
+ model::test_result_skipped,
+ "Required configuration property 'abcde' not defined"),
+ test_result_handle->test_result());
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__stacktrace);
+ATF_TEST_CASE_BODY(integration__stacktrace)
+{
+ utils::prepare_coredump_test(this);
+
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("unknown-dumps-core").build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "unknown-dumps-core", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_failed,
+ F("Signal %s") % SIGABRT),
+ test_result_handle->test_result());
+ ATF_REQUIRE(!atf::utils::grep_file("attempting to gather stack trace",
+ result_handle->stdout_file().str()));
+ ATF_REQUIRE( atf::utils::grep_file("attempting to gather stack trace",
+ result_handle->stderr_file().str()));
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+/// Runs a test to verify the dumping of the list of existing files on failure.
+///
+/// \param test_case The name of the test case to invoke.
+/// \param exp_stderr Expected contents of stderr.
+static void
+do_check_list_files_on_failure(const char* test_case, const char* exp_stderr)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case(test_case).build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, test_case, user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ atf::utils::cat_file(result_handle->stdout_file().str(), "child stdout: ");
+ ATF_REQUIRE(atf::utils::compare_file(result_handle->stdout_file().str(),
+ ""));
+ atf::utils::cat_file(result_handle->stderr_file().str(), "child stderr: ");
+ ATF_REQUIRE(atf::utils::compare_file(result_handle->stderr_file().str(),
+ exp_stderr));
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_files_on_failure__none);
+ATF_TEST_CASE_BODY(integration__list_files_on_failure__none)
+{
+ do_check_list_files_on_failure("fail", "This should not be clobbered\n");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__list_files_on_failure__some);
+ATF_TEST_CASE_BODY(integration__list_files_on_failure__some)
+{
+ do_check_list_files_on_failure(
+ "create_files_and_fail",
+ "This should not be clobbered\n"
+ "Files left in work directory after failure: "
+ "dir1, first file, second-file\n");
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(integration__prevent_clobbering_control_files);
+ATF_TEST_CASE_BODY(integration__prevent_clobbering_control_files)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("delete_all").build_ptr();
+
+ const config::tree user_config = engine::empty_config();
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ (void)handle.spawn_test(program, "delete_all", user_config);
+
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
+ test_result_handle->test_result());
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(debug_test);
+ATF_TEST_CASE_BODY(debug_test)
+{
+ const model::test_program_ptr program = model::test_program_builder(
+ "mock", fs::path("the-program"), fs::current_path(), "the-suite")
+ .add_test_case("print_params").build_ptr();
+
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("test_suites.the-suite.one", "first variable");
+ user_config.set_string("test_suites.the-suite.two", "second variable");
+
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ const fs::path stdout_file("custom-stdout.txt");
+ const fs::path stderr_file("custom-stderr.txt");
+
+ scheduler::result_handle_ptr result_handle = handle.debug_test(
+ program, "print_params", user_config, stdout_file, stderr_file);
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+
+ ATF_REQUIRE_EQ(program, test_result_handle->test_program());
+ ATF_REQUIRE_EQ("print_params", test_result_handle->test_case_name());
+ ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"),
+ test_result_handle->test_result());
+
+ // The original output went to a file. It's only an artifact of
+ // debug_test() that we later get a copy in our own files.
+ ATF_REQUIRE(stdout_file != result_handle->stdout_file());
+ ATF_REQUIRE(stderr_file != result_handle->stderr_file());
+
+ result_handle->cleanup();
+ result_handle.reset();
+
+ handle.cleanup();
+
+ ATF_REQUIRE(atf::utils::compare_file(
+ stdout_file.str(),
+ "Test program: the-program\n"
+ "Test case: print_params\n"
+ "one=first variable\n"
+ "two=second variable\n"));
+ ATF_REQUIRE(atf::utils::compare_file(
+ stderr_file.str(), "stderr: print_params\n"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(ensure_valid_interface);
+ATF_TEST_CASE_BODY(ensure_valid_interface)
+{
+ scheduler::ensure_valid_interface("mock");
+
+ ATF_REQUIRE_THROW_RE(engine::error, "Unsupported test interface 'mock2'",
+ scheduler::ensure_valid_interface("mock2"));
+ scheduler::register_interface(
+ "mock2", std::shared_ptr< scheduler::interface >(new mock_interface()));
+ scheduler::ensure_valid_interface("mock2");
+
+ // Standard interfaces should not be present unless registered.
+ ATF_REQUIRE_THROW_RE(engine::error, "Unsupported test interface 'plain'",
+ scheduler::ensure_valid_interface("plain"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(registered_interface_names);
+ATF_TEST_CASE_BODY(registered_interface_names)
+{
+ std::set< std::string > exp_names;
+
+ exp_names.insert("mock");
+ ATF_REQUIRE_EQ(exp_names, scheduler::registered_interface_names());
+
+ scheduler::register_interface(
+ "mock2", std::shared_ptr< scheduler::interface >(new mock_interface()));
+ exp_names.insert("mock2");
+ ATF_REQUIRE_EQ(exp_names, scheduler::registered_interface_names());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(current_context);
+ATF_TEST_CASE_BODY(current_context)
+{
+ const model::context context = scheduler::current_context();
+ ATF_REQUIRE_EQ(fs::current_path(), context.cwd());
+ ATF_REQUIRE(utils::getallenv() == context.env());
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(generate_config__empty);
+ATF_TEST_CASE_BODY(generate_config__empty)
+{
+ const config::tree user_config = engine::empty_config();
+
+ const config::properties_map exp_props;
+
+ ATF_REQUIRE_EQ(exp_props,
+ scheduler::generate_config(user_config, "missing"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(generate_config__no_matches);
+ATF_TEST_CASE_BODY(generate_config__no_matches)
+{
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("architecture", "foo");
+ user_config.set_string("test_suites.one.var1", "value 1");
+
+ const config::properties_map exp_props;
+
+ ATF_REQUIRE_EQ(exp_props,
+ scheduler::generate_config(user_config, "two"));
+}
+
+
+ATF_TEST_CASE_WITHOUT_HEAD(generate_config__some_matches);
+ATF_TEST_CASE_BODY(generate_config__some_matches)
+{
+ std::vector< passwd::user > mock_users;
+ mock_users.push_back(passwd::user("nobody", 1234, 5678));
+ passwd::set_mock_users_for_testing(mock_users);
+
+ config::tree user_config = engine::empty_config();
+ user_config.set_string("architecture", "foo");
+ user_config.set_string("unprivileged_user", "nobody");
+ user_config.set_string("test_suites.one.var1", "value 1");
+ user_config.set_string("test_suites.two.var2", "value 2");
+
+ config::properties_map exp_props;
+ exp_props["unprivileged-user"] = "nobody";
+ exp_props["var1"] = "value 1";
+
+ ATF_REQUIRE_EQ(exp_props,
+ scheduler::generate_config(user_config, "one"));
+}
+
+
+ATF_INIT_TEST_CASES(tcs)
+{
+ scheduler::register_interface(
+ "mock", std::shared_ptr< scheduler::interface >(new mock_interface()));
+
+ ATF_ADD_TEST_CASE(tcs, integration__list_some);
+ ATF_ADD_TEST_CASE(tcs, integration__list_check_paths);
+ ATF_ADD_TEST_CASE(tcs, integration__list_timeout);
+ ATF_ADD_TEST_CASE(tcs, integration__list_fail);
+ ATF_ADD_TEST_CASE(tcs, integration__list_empty);
+
+ ATF_ADD_TEST_CASE(tcs, integration__run_one);
+ ATF_ADD_TEST_CASE(tcs, integration__run_many);
+
+ ATF_ADD_TEST_CASE(tcs, integration__run_check_paths);
+ ATF_ADD_TEST_CASE(tcs, integration__parameters_and_output);
+
+ ATF_ADD_TEST_CASE(tcs, integration__fake_result);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__head_skips);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_skips);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_ok__cleanup_bad);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_bad__cleanup_ok);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_bad__cleanup_bad);
+ ATF_ADD_TEST_CASE(tcs, integration__cleanup__timeout);
+ ATF_ADD_TEST_CASE(tcs, integration__check_requirements);
+ ATF_ADD_TEST_CASE(tcs, integration__stacktrace);
+ ATF_ADD_TEST_CASE(tcs, integration__list_files_on_failure__none);
+ ATF_ADD_TEST_CASE(tcs, integration__list_files_on_failure__some);
+ ATF_ADD_TEST_CASE(tcs, integration__prevent_clobbering_control_files);
+
+ ATF_ADD_TEST_CASE(tcs, debug_test);
+
+ ATF_ADD_TEST_CASE(tcs, ensure_valid_interface);
+ ATF_ADD_TEST_CASE(tcs, registered_interface_names);
+
+ ATF_ADD_TEST_CASE(tcs, current_context);
+
+ ATF_ADD_TEST_CASE(tcs, generate_config__empty);
+ ATF_ADD_TEST_CASE(tcs, generate_config__no_matches);
+ ATF_ADD_TEST_CASE(tcs, generate_config__some_matches);
+}