aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/run_tests.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/run_tests.cpp')
-rw-r--r--drivers/run_tests.cpp344
1 files changed, 344 insertions, 0 deletions
diff --git a/drivers/run_tests.cpp b/drivers/run_tests.cpp
new file mode 100644
index 000000000000..d92940005242
--- /dev/null
+++ b/drivers/run_tests.cpp
@@ -0,0 +1,344 @@
+// Copyright 2011 The Kyua Authors.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of Google Inc. nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "drivers/run_tests.hpp"
+
+#include <utility>
+
+#include "engine/config.hpp"
+#include "engine/filters.hpp"
+#include "engine/kyuafile.hpp"
+#include "engine/scanner.hpp"
+#include "engine/scheduler.hpp"
+#include "model/context.hpp"
+#include "model/metadata.hpp"
+#include "model/test_case.hpp"
+#include "model/test_program.hpp"
+#include "model/test_result.hpp"
+#include "store/write_backend.hpp"
+#include "store/write_transaction.hpp"
+#include "utils/config/tree.ipp"
+#include "utils/datetime.hpp"
+#include "utils/defs.hpp"
+#include "utils/format/macros.hpp"
+#include "utils/logging/macros.hpp"
+#include "utils/noncopyable.hpp"
+#include "utils/optional.ipp"
+#include "utils/passwd.hpp"
+#include "utils/text/operations.ipp"
+
+namespace config = utils::config;
+namespace datetime = utils::datetime;
+namespace fs = utils::fs;
+namespace passwd = utils::passwd;
+namespace scheduler = engine::scheduler;
+namespace text = utils::text;
+
+using utils::none;
+using utils::optional;
+
+
+namespace {
+
+
+/// Map of test program identifiers (relative paths) to their identifiers in the
+/// database. We need to keep this in memory because test programs can be
+/// returned by the scanner in any order, and we only want to put each test
+/// program once.
+typedef std::map< fs::path, int64_t > path_to_id_map;
+
+
+/// Map of in-flight PIDs to their corresponding test case IDs.
+typedef std::map< int, int64_t > pid_to_id_map;
+
+
+/// Pair of PID to a test case ID.
+typedef pid_to_id_map::value_type pid_and_id_pair;
+
+
+/// Puts a test program in the store and returns its identifier.
+///
+/// This function is idempotent: we maintain a side cache of already-put test
+/// programs so that we can return their identifiers without having to put them
+/// again.
+/// TODO(jmmv): It's possible that the store module should offer this
+/// functionality and not have to do this ourselves here.
+///
+/// \param test_program The test program being put.
+/// \param [in,out] tx Writable transaction on the store.
+/// \param [in,out] ids_cache Cache of already-put test programs.
+///
+/// \return A test program identifier.
+static int64_t
+find_test_program_id(const model::test_program_ptr test_program,
+ store::write_transaction& tx,
+ path_to_id_map& ids_cache)
+{
+ const fs::path& key = test_program->relative_path();
+ std::map< fs::path, int64_t >::const_iterator iter = ids_cache.find(key);
+ if (iter == ids_cache.end()) {
+ const int64_t id = tx.put_test_program(*test_program);
+ ids_cache.insert(std::make_pair(key, id));
+ return id;
+ } else {
+ return (*iter).second;
+ }
+}
+
+
+/// Stores the result of an execution in the database.
+///
+/// \param test_case_id Identifier of the test case in the database.
+/// \param result The result of the execution.
+/// \param [in,out] tx Writable transaction where to store the result data.
+static void
+put_test_result(const int64_t test_case_id,
+ const scheduler::test_result_handle& result,
+ store::write_transaction& tx)
+{
+ tx.put_result(result.test_result(), test_case_id,
+ result.start_time(), result.end_time());
+ tx.put_test_case_file("__STDOUT__", result.stdout_file(), test_case_id);
+ tx.put_test_case_file("__STDERR__", result.stderr_file(), test_case_id);
+
+}
+
+
+/// Cleans up a test case and folds any errors into the test result.
+///
+/// \param handle The result handle for the test.
+///
+/// \return The test result if the cleanup succeeds; a broken test result
+/// otherwise.
+model::test_result
+safe_cleanup(scheduler::test_result_handle handle) throw()
+{
+ try {
+ handle.cleanup();
+ return handle.test_result();
+ } catch (const std::exception& e) {
+ return model::test_result(
+ model::test_result_broken,
+ F("Failed to clean up test case's work directory %s: %s") %
+ handle.work_directory() % e.what());
+ }
+}
+
+
+/// Starts a test asynchronously.
+///
+/// \param handle Scheduler handle.
+/// \param match Test program and test case to start.
+/// \param [in,out] tx Writable transaction to obtain test IDs.
+/// \param [in,out] ids_cache Cache of already-put test cases.
+/// \param user_config The end-user configuration properties.
+/// \param hooks The hooks for this execution.
+///
+/// \returns The PID for the started test and the test case's identifier in the
+/// store.
+pid_and_id_pair
+start_test(scheduler::scheduler_handle& handle,
+ const engine::scan_result& match,
+ store::write_transaction& tx,
+ path_to_id_map& ids_cache,
+ const config::tree& user_config,
+ drivers::run_tests::base_hooks& hooks)
+{
+ const model::test_program_ptr test_program = match.first;
+ const std::string& test_case_name = match.second;
+
+ hooks.got_test_case(*test_program, test_case_name);
+
+ const int64_t test_program_id = find_test_program_id(
+ test_program, tx, ids_cache);
+ const int64_t test_case_id = tx.put_test_case(
+ *test_program, test_case_name, test_program_id);
+
+ const scheduler::exec_handle exec_handle = handle.spawn_test(
+ test_program, test_case_name, user_config);
+ return std::make_pair(exec_handle, test_case_id);
+}
+
+
+/// Processes the completion of a test.
+///
+/// \param [in,out] result_handle The completion handle of the test subprocess.
+/// \param test_case_id Identifier of the test case as returned by start_test().
+/// \param [in,out] tx Writable transaction to put the test results.
+/// \param hooks The hooks for this execution.
+///
+/// \post result_handle is cleaned up. The caller cannot clean it up again.
+void
+finish_test(scheduler::result_handle_ptr result_handle,
+ const int64_t test_case_id,
+ store::write_transaction& tx,
+ drivers::run_tests::base_hooks& hooks)
+{
+ const scheduler::test_result_handle* test_result_handle =
+ dynamic_cast< const scheduler::test_result_handle* >(
+ result_handle.get());
+
+ put_test_result(test_case_id, *test_result_handle, tx);
+
+ const model::test_result test_result = safe_cleanup(*test_result_handle);
+ hooks.got_result(
+ *test_result_handle->test_program(),
+ test_result_handle->test_case_name(),
+ test_result_handle->test_result(),
+ result_handle->end_time() - result_handle->start_time());
+}
+
+
+/// Extracts the keys of a pid_to_id_map and returns them as a string.
+///
+/// \param map The PID to test ID map from which to get the PIDs.
+///
+/// \return A user-facing string with the collection of PIDs.
+static std::string
+format_pids(const pid_to_id_map& map)
+{
+ std::set< pid_to_id_map::key_type > pids;
+ for (pid_to_id_map::const_iterator iter = map.begin(); iter != map.end();
+ ++iter) {
+ pids.insert(iter->first);
+ }
+ return text::join(pids, ",");
+}
+
+
+} // anonymous namespace
+
+
+/// Pure abstract destructor.
+drivers::run_tests::base_hooks::~base_hooks(void)
+{
+}
+
+
+/// Executes the operation.
+///
+/// \param kyuafile_path The path to the Kyuafile to be loaded.
+/// \param build_root If not none, path to the built test programs.
+/// \param store_path The path to the store to be used.
+/// \param filters The test case filters as provided by the user.
+/// \param user_config The end-user configuration properties.
+/// \param hooks The hooks for this execution.
+///
+/// \returns A structure with all results computed by this driver.
+drivers::run_tests::result
+drivers::run_tests::drive(const fs::path& kyuafile_path,
+ const optional< fs::path > build_root,
+ const fs::path& store_path,
+ const std::set< engine::test_filter >& filters,
+ const config::tree& user_config,
+ base_hooks& hooks)
+{
+ scheduler::scheduler_handle handle = scheduler::setup();
+
+ const engine::kyuafile kyuafile = engine::kyuafile::load(
+ kyuafile_path, build_root, user_config, handle);
+ store::write_backend db = store::write_backend::open_rw(store_path);
+ store::write_transaction tx = db.start_write();
+
+ {
+ const model::context context = scheduler::current_context();
+ (void)tx.put_context(context);
+ }
+
+ engine::scanner scanner(kyuafile.test_programs(), filters);
+
+ path_to_id_map ids_cache;
+ pid_to_id_map in_flight;
+ std::vector< engine::scan_result > exclusive_tests;
+
+ const std::size_t slots = user_config.lookup< config::positive_int_node >(
+ "parallelism");
+ INV(slots >= 1);
+ do {
+ INV(in_flight.size() <= slots);
+
+ // Spawn as many jobs as needed to fill our execution slots. We do this
+ // first with the assumption that the spawning is faster than any single
+ // job, so we want to keep as many jobs in the background as possible.
+ while (in_flight.size() < slots) {
+ optional< engine::scan_result > match = scanner.yield();
+ if (!match)
+ break;
+ const model::test_program_ptr test_program = match.get().first;
+ const std::string& test_case_name = match.get().second;
+
+ const model::test_case& test_case = test_program->find(
+ test_case_name);
+ if (test_case.get_metadata().is_exclusive()) {
+ // Exclusive tests get processed later, separately.
+ exclusive_tests.push_back(match.get());
+ continue;
+ }
+
+ const pid_and_id_pair pid_id = start_test(
+ handle, match.get(), tx, ids_cache, user_config, hooks);
+ INV_MSG(in_flight.find(pid_id.first) == in_flight.end(),
+ F("Spawned test has PID of still-tracked process %s") %
+ pid_id.first);
+ in_flight.insert(pid_id);
+ }
+
+ // If there are any used slots, consume any at random and return the
+ // result. We consume slots one at a time to give preference to the
+ // spawning of new tests as detailed above.
+ if (!in_flight.empty()) {
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+
+ const pid_to_id_map::iterator iter = in_flight.find(
+ result_handle->original_pid());
+ INV_MSG(iter != in_flight.end(),
+ F("Lost track of in-flight PID %s; tracking %s") %
+ result_handle->original_pid() % format_pids(in_flight));
+ const int64_t test_case_id = (*iter).second;
+ in_flight.erase(iter);
+
+ finish_test(result_handle, test_case_id, tx, hooks);
+ }
+ } while (!in_flight.empty() || !scanner.done());
+
+ // Run any exclusive tests that we spotted earlier sequentially.
+ for (std::vector< engine::scan_result >::const_iterator
+ iter = exclusive_tests.begin(); iter != exclusive_tests.end();
+ ++iter) {
+ const pid_and_id_pair data = start_test(
+ handle, *iter, tx, ids_cache, user_config, hooks);
+ scheduler::result_handle_ptr result_handle = handle.wait_any();
+ finish_test(result_handle, data.second, tx, hooks);
+ }
+
+ tx.commit();
+
+ handle.cleanup();
+
+ return result(scanner.unused_filters());
+}