diff options
author | Brooks Davis <brooks@FreeBSD.org> | 2020-03-17 16:56:50 +0000 |
---|---|---|
committer | Brooks Davis <brooks@FreeBSD.org> | 2020-03-17 16:56:50 +0000 |
commit | 08334c51dbb99d9ecd2bb86a2d94ed06da9e167a (patch) | |
tree | c43eb24d59bd5c963583a5190caef80fc8387322 /drivers | |
download | src-vendor/kyua.tar.gz src-vendor/kyua.zip |
Import the kyua testing framework for infrastructure softwarevendor/kyua/0.13-a685f91vendor/kyua
Imported at 0.13 plus assumulated changes to git hash a685f91.
Obtained from: https://github.com/jmmv/kyua
Sponsored by: DARPA
Notes
Notes:
svn path=/vendor/kyua/dist/; revision=359042
svn path=/vendor/kyua/0.13-a685f91/; revision=359043; tag=vendor/kyua/0.13-a685f91
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/Kyuafile | 7 | ||||
-rw-r--r-- | drivers/Makefile.am.inc | 72 | ||||
-rw-r--r-- | drivers/debug_test.cpp | 109 | ||||
-rw-r--r-- | drivers/debug_test.hpp | 79 | ||||
-rw-r--r-- | drivers/list_tests.cpp | 84 | ||||
-rw-r--r-- | drivers/list_tests.hpp | 92 | ||||
-rw-r--r-- | drivers/list_tests_helpers.cpp | 98 | ||||
-rw-r--r-- | drivers/list_tests_test.cpp | 287 | ||||
-rw-r--r-- | drivers/report_junit.cpp | 258 | ||||
-rw-r--r-- | drivers/report_junit.hpp | 75 | ||||
-rw-r--r-- | drivers/report_junit_test.cpp | 415 | ||||
-rw-r--r-- | drivers/run_tests.cpp | 344 | ||||
-rw-r--r-- | drivers/run_tests.hpp | 106 | ||||
-rw-r--r-- | drivers/scan_results.cpp | 107 | ||||
-rw-r--r-- | drivers/scan_results.hpp | 105 | ||||
-rw-r--r-- | drivers/scan_results_test.cpp | 258 |
16 files changed, 2496 insertions, 0 deletions
diff --git a/drivers/Kyuafile b/drivers/Kyuafile new file mode 100644 index 000000000000..35902737c943 --- /dev/null +++ b/drivers/Kyuafile @@ -0,0 +1,7 @@ +syntax(2) + +test_suite("kyua") + +atf_test_program{name="list_tests_test"} +atf_test_program{name="report_junit_test"} +atf_test_program{name="scan_results_test"} diff --git a/drivers/Makefile.am.inc b/drivers/Makefile.am.inc new file mode 100644 index 000000000000..0bd5a08209de --- /dev/null +++ b/drivers/Makefile.am.inc @@ -0,0 +1,72 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +DRIVERS_CFLAGS = $(ENGINE_CFLAGS) $(STORE_CFLAGS) $(MODEL_CFLAGS) \ + $(UTILS_CFLAGS) +DRIVERS_LIBS = libdrivers.a $(ENGINE_LIBS) $(STORE_LIBS) $(MODEL_LIBS) \ + $(UTILS_LIBS) + +noinst_LIBRARIES += libdrivers.a +libdrivers_a_CPPFLAGS = $(DRIVERS_CFLAGS) +libdrivers_a_SOURCES = drivers/debug_test.cpp +libdrivers_a_SOURCES += drivers/debug_test.hpp +libdrivers_a_SOURCES += drivers/list_tests.cpp +libdrivers_a_SOURCES += drivers/list_tests.hpp +libdrivers_a_SOURCES += drivers/report_junit.cpp +libdrivers_a_SOURCES += drivers/report_junit.hpp +libdrivers_a_SOURCES += drivers/run_tests.cpp +libdrivers_a_SOURCES += drivers/run_tests.hpp +libdrivers_a_SOURCES += drivers/scan_results.cpp +libdrivers_a_SOURCES += drivers/scan_results.hpp + +if WITH_ATF +tests_driversdir = $(pkgtestsdir)/drivers + +tests_drivers_DATA = drivers/Kyuafile +EXTRA_DIST += $(tests_drivers_DATA) + +tests_drivers_PROGRAMS = drivers/list_tests_helpers +drivers_list_tests_helpers_SOURCES = drivers/list_tests_helpers.cpp +drivers_list_tests_helpers_CXXFLAGS = $(ATF_CXX_CFLAGS) +drivers_list_tests_helpers_LDADD = $(ATF_CXX_LIBS) + +tests_drivers_PROGRAMS += drivers/list_tests_test +drivers_list_tests_test_SOURCES = drivers/list_tests_test.cpp +drivers_list_tests_test_CXXFLAGS = $(DRIVERS_CFLAGS) $(ATF_CXX_CFLAGS) +drivers_list_tests_test_LDADD = $(DRIVERS_LIBS) $(ATF_CXX_LIBS) + +tests_drivers_PROGRAMS += drivers/report_junit_test +drivers_report_junit_test_SOURCES = drivers/report_junit_test.cpp +drivers_report_junit_test_CXXFLAGS = $(DRIVERS_CFLAGS) $(ATF_CXX_CFLAGS) +drivers_report_junit_test_LDADD = $(DRIVERS_LIBS) $(ATF_CXX_LIBS) + +tests_drivers_PROGRAMS += drivers/scan_results_test +drivers_scan_results_test_SOURCES = drivers/scan_results_test.cpp +drivers_scan_results_test_CXXFLAGS = $(DRIVERS_CFLAGS) $(ATF_CXX_CFLAGS) +drivers_scan_results_test_LDADD = $(DRIVERS_LIBS) $(ATF_CXX_LIBS) +endif diff --git a/drivers/debug_test.cpp b/drivers/debug_test.cpp new file mode 100644 index 000000000000..0717a9ad9419 --- /dev/null +++ b/drivers/debug_test.cpp @@ -0,0 +1,109 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "drivers/debug_test.hpp" + +#include <stdexcept> +#include <utility> + +#include "engine/filters.hpp" +#include "engine/kyuafile.hpp" +#include "engine/scanner.hpp" +#include "engine/scheduler.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/defs.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/auto_cleaners.hpp" +#include "utils/optional.ipp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace scheduler = engine::scheduler; + +using utils::optional; + + +/// Executes the operation. +/// +/// \param kyuafile_path The path to the Kyuafile to be loaded. +/// \param build_root If not none, path to the built test programs. +/// \param filter The test case filter to locate the test to debug. +/// \param user_config The end-user configuration properties. +/// \param stdout_path The name of the file into which to store the test case +/// stdout. +/// \param stderr_path The name of the file into which to store the test case +/// stderr. +/// +/// \returns A structure with all results computed by this driver. +drivers::debug_test::result +drivers::debug_test::drive(const fs::path& kyuafile_path, + const optional< fs::path > build_root, + const engine::test_filter& filter, + const config::tree& user_config, + const fs::path& stdout_path, + const fs::path& stderr_path) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + const engine::kyuafile kyuafile = engine::kyuafile::load( + kyuafile_path, build_root, user_config, handle); + std::set< engine::test_filter > filters; + filters.insert(filter); + + engine::scanner scanner(kyuafile.test_programs(), filters); + optional< engine::scan_result > match; + while (!match && !scanner.done()) { + match = scanner.yield(); + } + if (!match) { + throw std::runtime_error(F("Unknown test case '%s'") % filter.str()); + } else if (!scanner.done()) { + throw std::runtime_error(F("The filter '%s' matches more than one test " + "case") % filter.str()); + } + INV(match && scanner.done()); + const model::test_program_ptr test_program = match.get().first; + const std::string& test_case_name = match.get().second; + + scheduler::result_handle_ptr result_handle = handle.debug_test( + test_program, test_case_name, user_config, + stdout_path, stderr_path); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + const model::test_result test_result = test_result_handle->test_result(); + result_handle->cleanup(); + + handle.check_interrupt(); + handle.cleanup(); + + return result(engine::test_filter( + test_program->relative_path(), test_case_name), test_result); +} diff --git a/drivers/debug_test.hpp b/drivers/debug_test.hpp new file mode 100644 index 000000000000..cbaa2f6acea0 --- /dev/null +++ b/drivers/debug_test.hpp @@ -0,0 +1,79 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file drivers/debug_test.hpp +/// Driver to run a single test in a controlled manner. +/// +/// This driver module implements the logic to execute a particular test +/// with hooks into the runtime procedure. This is to permit debugging the +/// behavior of the test. + +#if !defined(DRIVERS_DEBUG_TEST_HPP) +#define DRIVERS_DEBUG_TEST_HPP + +#include "engine/filters.hpp" +#include "model/test_result.hpp" +#include "utils/config/tree_fwd.hpp" +#include "utils/fs/path_fwd.hpp" +#include "utils/optional_fwd.hpp" + +namespace drivers { +namespace debug_test { + + +/// Tuple containing the results of this driver. +class result { +public: + /// A filter matching the executed test case only. + engine::test_filter test_case; + + /// The result of the test case. + model::test_result test_result; + + /// Initializer for the tuple's fields. + /// + /// \param test_case_ The matched test case. + /// \param test_result_ The result of the test case. + result(const engine::test_filter& test_case_, + const model::test_result& test_result_) : + test_case(test_case_), + test_result(test_result_) + { + } +}; + + +result drive(const utils::fs::path&, const utils::optional< utils::fs::path >, + const engine::test_filter&, const utils::config::tree&, + const utils::fs::path&, const utils::fs::path&); + + +} // namespace debug_test +} // namespace drivers + +#endif // !defined(DRIVERS_DEBUG_TEST_HPP) diff --git a/drivers/list_tests.cpp b/drivers/list_tests.cpp new file mode 100644 index 000000000000..b56706d30b93 --- /dev/null +++ b/drivers/list_tests.cpp @@ -0,0 +1,84 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "drivers/list_tests.hpp" + +#include "engine/exceptions.hpp" +#include "engine/filters.hpp" +#include "engine/kyuafile.hpp" +#include "engine/scanner.hpp" +#include "engine/scheduler.hpp" +#include "model/test_program.hpp" +#include "utils/optional.ipp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace scheduler = engine::scheduler; + +using utils::optional; + + +/// Pure abstract destructor. +drivers::list_tests::base_hooks::~base_hooks(void) +{ +} + + +/// Executes the operation. +/// +/// \param kyuafile_path The path to the Kyuafile to be loaded. +/// \param build_root If not none, path to the built test programs. +/// \param filters The test case filters as provided by the user. +/// \param user_config The end-user configuration properties. +/// \param hooks The hooks for this execution. +/// +/// \returns A structure with all results computed by this driver. +drivers::list_tests::result +drivers::list_tests::drive(const fs::path& kyuafile_path, + const optional< fs::path > build_root, + const std::set< engine::test_filter >& filters, + const config::tree& user_config, + base_hooks& hooks) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + const engine::kyuafile kyuafile = engine::kyuafile::load( + kyuafile_path, build_root, user_config, handle); + + engine::scanner scanner(kyuafile.test_programs(), filters); + + while (!scanner.done()) { + const optional< engine::scan_result > result = scanner.yield(); + INV(result); + hooks.got_test_case(*result.get().first, result.get().second); + } + + handle.cleanup(); + + return result(scanner.unused_filters()); +} diff --git a/drivers/list_tests.hpp b/drivers/list_tests.hpp new file mode 100644 index 000000000000..6b1257e41b22 --- /dev/null +++ b/drivers/list_tests.hpp @@ -0,0 +1,92 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file drivers/list_tests.hpp +/// Driver to obtain a list of test cases out of a test suite. +/// +/// This driver module implements the logic to extract a list of test cases out +/// of a particular test suite. + +#if !defined(DRIVERS_LIST_TESTS_HPP) +#define DRIVERS_LIST_TESTS_HPP + +#include <set> +#include <string> + +#include "engine/filters_fwd.hpp" +#include "model/test_program_fwd.hpp" +#include "utils/config/tree_fwd.hpp" +#include "utils/fs/path_fwd.hpp" +#include "utils/optional_fwd.hpp" + +namespace drivers { +namespace list_tests { + + +/// Abstract definition of the hooks for this driver. +class base_hooks { +public: + virtual ~base_hooks(void) = 0; + + /// Called when a test case is identified in a test suite. + /// + /// \param test_program The test program containing the test case. + /// \param test_case_name The name of the located test case. + virtual void got_test_case(const model::test_program& test_program, + const std::string& test_case_name) = 0; +}; + + +/// Tuple containing the results of this driver. +class result { +public: + /// Filters that did not match any available test case. + /// + /// The presence of any filters here probably indicates a usage error. If a + /// test filter does not match any test case, it is probably a typo. + std::set< engine::test_filter > unused_filters; + + /// Initializer for the tuple's fields. + /// + /// \param unused_filters_ The filters that did not match any test case. + result(const std::set< engine::test_filter >& unused_filters_) : + unused_filters(unused_filters_) + { + } +}; + + +result drive(const utils::fs::path&, const utils::optional< utils::fs::path >, + const std::set< engine::test_filter >&, + const utils::config::tree&, base_hooks&); + + +} // namespace list_tests +} // namespace drivers + +#endif // !defined(DRIVERS_LIST_TESTS_HPP) diff --git a/drivers/list_tests_helpers.cpp b/drivers/list_tests_helpers.cpp new file mode 100644 index 000000000000..2b40b1e11db1 --- /dev/null +++ b/drivers/list_tests_helpers.cpp @@ -0,0 +1,98 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <cstdlib> + +#include <atf-c++.hpp> + +#include "utils/test_utils.ipp" + + +ATF_TEST_CASE(config_in_head); +ATF_TEST_CASE_HEAD(config_in_head) +{ + if (has_config_var("the-variable")) { + set_md_var("descr", "the-variable is " + + get_config_var("the-variable")); + } +} +ATF_TEST_CASE_BODY(config_in_head) +{ + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE(crash_list); +ATF_TEST_CASE_HEAD(crash_list) +{ + utils::abort_without_coredump(); +} +ATF_TEST_CASE_BODY(crash_list) +{ + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(no_properties); +ATF_TEST_CASE_BODY(no_properties) +{ + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE(some_properties); +ATF_TEST_CASE_HEAD(some_properties) +{ + set_md_var("descr", "This is a description"); + set_md_var("require.progs", "non-existent /bin/ls"); +} +ATF_TEST_CASE_BODY(some_properties) +{ + utils::abort_without_coredump(); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + std::string enabled; + + const char* tests = std::getenv("TESTS"); + if (tests == NULL) + enabled = "config_in_head crash_list no_properties some_properties"; + else + enabled = tests; + + if (enabled.find("config_in_head") != std::string::npos) + ATF_ADD_TEST_CASE(tcs, config_in_head); + if (enabled.find("crash_list") != std::string::npos) + ATF_ADD_TEST_CASE(tcs, crash_list); + if (enabled.find("no_properties") != std::string::npos) + ATF_ADD_TEST_CASE(tcs, no_properties); + if (enabled.find("some_properties") != std::string::npos) + ATF_ADD_TEST_CASE(tcs, some_properties); +} diff --git a/drivers/list_tests_test.cpp b/drivers/list_tests_test.cpp new file mode 100644 index 000000000000..752b251052ad --- /dev/null +++ b/drivers/list_tests_test.cpp @@ -0,0 +1,287 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "drivers/list_tests.hpp" + +extern "C" { +#include <sys/stat.h> + +#include <unistd.h> +} + +#include <map> +#include <set> +#include <string> + +#include <atf-c++.hpp> + +#include "cli/cmd_list.hpp" +#include "cli/common.ipp" +#include "engine/atf.hpp" +#include "engine/config.hpp" +#include "engine/exceptions.hpp" +#include "engine/filters.hpp" +#include "engine/scheduler.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "utils/config/tree.ipp" +#include "utils/env.hpp" +#include "utils/format/macros.hpp" +#include "utils/optional.ipp" +#include "utils/test_utils.ipp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace scheduler = engine::scheduler; + +using utils::none; +using utils::optional; + + +namespace { + + +/// Gets the path to the helpers for this test program. +/// +/// \param test_case A pointer to the currently running test case. +/// +/// \return The path to the helpers binary. +static fs::path +helpers(const atf::tests::tc* test_case) +{ + return fs::path(test_case->get_config_var("srcdir")) / + "list_tests_helpers"; +} + + +/// Hooks to capture the incremental listing of test cases. +class capture_hooks : public drivers::list_tests::base_hooks { +public: + /// Set of the listed test cases in a program:test_case form. + std::set< std::string > test_cases; + + /// Set of the listed test cases in a program:test_case form. + std::map< std::string, model::metadata > metadatas; + + /// Called when a test case is identified in a test suite. + /// + /// \param test_program The test program containing the test case. + /// \param test_case_name The name of the located test case. + virtual void + got_test_case(const model::test_program& test_program, + const std::string& test_case_name) + { + const std::string ident = F("%s:%s") % + test_program.relative_path() % test_case_name; + test_cases.insert(ident); + + metadatas.insert(std::map< std::string, model::metadata >::value_type( + ident, test_program.find(test_case_name).get_metadata())); + } +}; + + +/// Creates a mock test suite. +/// +/// \param tc Pointer to the caller test case; needed to obtain the srcdir +/// variable of the caller. +/// \param source_root Basename of the directory that will contain the +/// Kyuafiles. +/// \param build_root Basename of the directory that will contain the test +/// programs. May or may not be the same as source_root. +static void +create_helpers(const atf::tests::tc* tc, const fs::path& source_root, + const fs::path& build_root) +{ + ATF_REQUIRE(::mkdir(source_root.c_str(), 0755) != -1); + ATF_REQUIRE(::mkdir((source_root / "dir").c_str(), 0755) != -1); + if (source_root != build_root) { + ATF_REQUIRE(::mkdir(build_root.c_str(), 0755) != -1); + ATF_REQUIRE(::mkdir((build_root / "dir").c_str(), 0755) != -1); + } + ATF_REQUIRE(::symlink(helpers(tc).c_str(), + (build_root / "dir/program").c_str()) != -1); + + atf::utils::create_file( + (source_root / "Kyuafile").str(), + "syntax(2)\n" + "include('dir/Kyuafile')\n"); + + atf::utils::create_file( + (source_root / "dir/Kyuafile").str(), + "syntax(2)\n" + "atf_test_program{name='program', test_suite='suite-name'}\n"); +} + + +/// Runs the mock test suite. +/// +/// \param source_root Path to the directory that contains the Kyuafiles. +/// \param build_root If not none, path to the directory that contains the test +/// programs. +/// \param hooks The hooks to use during the listing. +/// \param filter_program If not null, the filter on the test program name. +/// \param filter_test_case If not null, the filter on the test case name. +/// \param the_variable If not null, the value to pass to the test program as +/// its "the-variable" configuration property. +/// +/// \return The result data of the driver. +static drivers::list_tests::result +run_helpers(const fs::path& source_root, + const optional< fs::path > build_root, + drivers::list_tests::base_hooks& hooks, + const char* filter_program = NULL, + const char* filter_test_case = NULL, + const char* the_variable = NULL) +{ + std::set< engine::test_filter > filters; + if (filter_program != NULL && filter_test_case != NULL) + filters.insert(engine::test_filter(fs::path(filter_program), + filter_test_case)); + + config::tree user_config = engine::empty_config(); + if (the_variable != NULL) { + user_config.set_string("test_suites.suite-name.the-variable", + the_variable); + } + + return drivers::list_tests::drive(source_root / "Kyuafile", build_root, + filters, user_config, hooks); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(one_test_case); +ATF_TEST_CASE_BODY(one_test_case) +{ + utils::setenv("TESTS", "some_properties"); + capture_hooks hooks; + create_helpers(this, fs::path("root"), fs::path("root")); + run_helpers(fs::path("root"), none, hooks); + + std::set< std::string > exp_test_cases; + exp_test_cases.insert("dir/program:some_properties"); + ATF_REQUIRE(exp_test_cases == hooks.test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(many_test_cases); +ATF_TEST_CASE_BODY(many_test_cases) +{ + utils::setenv("TESTS", "no_properties some_properties"); + capture_hooks hooks; + create_helpers(this, fs::path("root"), fs::path("root")); + run_helpers(fs::path("root"), none, hooks); + + std::set< std::string > exp_test_cases; + exp_test_cases.insert("dir/program:no_properties"); + exp_test_cases.insert("dir/program:some_properties"); + ATF_REQUIRE(exp_test_cases == hooks.test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(filter_match); +ATF_TEST_CASE_BODY(filter_match) +{ + utils::setenv("TESTS", "no_properties some_properties"); + capture_hooks hooks; + create_helpers(this, fs::path("root"), fs::path("root")); + run_helpers(fs::path("root"), none, hooks, "dir/program", + "some_properties"); + + std::set< std::string > exp_test_cases; + exp_test_cases.insert("dir/program:some_properties"); + ATF_REQUIRE(exp_test_cases == hooks.test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(build_root); +ATF_TEST_CASE_BODY(build_root) +{ + utils::setenv("TESTS", "no_properties some_properties"); + capture_hooks hooks; + create_helpers(this, fs::path("source"), fs::path("build")); + run_helpers(fs::path("source"), utils::make_optional(fs::path("build")), + hooks); + + std::set< std::string > exp_test_cases; + exp_test_cases.insert("dir/program:no_properties"); + exp_test_cases.insert("dir/program:some_properties"); + ATF_REQUIRE(exp_test_cases == hooks.test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(config_in_head); +ATF_TEST_CASE_BODY(config_in_head) +{ + utils::setenv("TESTS", "config_in_head"); + capture_hooks hooks; + create_helpers(this, fs::path("source"), fs::path("build")); + run_helpers(fs::path("source"), utils::make_optional(fs::path("build")), + hooks, NULL, NULL, "magic value"); + + std::set< std::string > exp_test_cases; + exp_test_cases.insert("dir/program:config_in_head"); + ATF_REQUIRE(exp_test_cases == hooks.test_cases); + + const model::metadata& metadata = hooks.metadatas.find( + "dir/program:config_in_head")->second; + ATF_REQUIRE_EQ("the-variable is magic value", metadata.description()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(crash); +ATF_TEST_CASE_BODY(crash) +{ + utils::setenv("TESTS", "crash_list some_properties"); + capture_hooks hooks; + create_helpers(this, fs::path("root"), fs::path("root")); + run_helpers(fs::path("root"), none, hooks, "dir/program"); + + std::set< std::string > exp_test_cases; + exp_test_cases.insert("dir/program:__test_cases_list__"); + ATF_REQUIRE(exp_test_cases == hooks.test_cases); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + scheduler::register_interface( + "atf", std::shared_ptr< scheduler::interface >( + new engine::atf_interface())); + + ATF_ADD_TEST_CASE(tcs, one_test_case); + ATF_ADD_TEST_CASE(tcs, many_test_cases); + ATF_ADD_TEST_CASE(tcs, filter_match); + ATF_ADD_TEST_CASE(tcs, build_root); + ATF_ADD_TEST_CASE(tcs, config_in_head); + ATF_ADD_TEST_CASE(tcs, crash); +} diff --git a/drivers/report_junit.cpp b/drivers/report_junit.cpp new file mode 100644 index 000000000000..4c14d535675f --- /dev/null +++ b/drivers/report_junit.cpp @@ -0,0 +1,258 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "drivers/report_junit.hpp" + +#include <algorithm> + +#include "model/context.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "model/types.hpp" +#include "store/read_transaction.hpp" +#include "utils/datetime.hpp" +#include "utils/defs.hpp" +#include "utils/format/macros.hpp" +#include "utils/text/operations.hpp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace text = utils::text; + + +/// Converts a test program name into a class-like name. +/// +/// \param test_program Test program from which to extract the name. +/// +/// \return A class-like representation of the test program's identifier. +std::string +drivers::junit_classname(const model::test_program& test_program) +{ + std::string classname = test_program.relative_path().str(); + std::replace(classname.begin(), classname.end(), '/', '.'); + return classname; +} + + +/// Converts a test case's duration to a second-based representation. +/// +/// \param delta The duration to convert. +/// +/// \return A second-based with millisecond-precision representation of the +/// input duration. +std::string +drivers::junit_duration(const datetime::delta& delta) +{ + return F("%.3s") % (delta.seconds + (delta.useconds / 1000000.0)); +} + + +/// String to prepend to the formatted test case metadata. +const char* const drivers::junit_metadata_header = + "Test case metadata\n" + "------------------\n" + "\n"; + + +/// String to prepend to the formatted test case timing details. +const char* const drivers::junit_timing_header = + "\n" + "Timing information\n" + "------------------\n" + "\n"; + + +/// String to append to the formatted test case metadata. +const char* const drivers::junit_stderr_header = + "\n" + "Original stderr\n" + "---------------\n" + "\n"; + + +/// Formats a test's metadata for recording in stderr. +/// +/// \param metadata The metadata to format. +/// +/// \return A string with the metadata contents that can be prepended to the +/// original test's stderr. +std::string +drivers::junit_metadata(const model::metadata& metadata) +{ + const model::properties_map props = metadata.to_properties(); + if (props.empty()) + return ""; + + std::ostringstream output; + output << junit_metadata_header; + for (model::properties_map::const_iterator iter = props.begin(); + iter != props.end(); ++iter) { + if ((*iter).second.empty()) { + output << F("%s is empty\n") % (*iter).first; + } else { + output << F("%s = %s\n") % (*iter).first % (*iter).second; + } + } + return output.str(); +} + + +/// Formats a test's timing information for recording in stderr. +/// +/// \param start_time The start time of the test. +/// \param end_time The end time of the test. +/// +/// \return A string with the timing information that can be prepended to the +/// original test's stderr. +std::string +drivers::junit_timing(const datetime::timestamp& start_time, + const datetime::timestamp& end_time) +{ + std::ostringstream output; + output << junit_timing_header; + output << F("Start time: %s\n") % start_time.to_iso8601_in_utc(); + output << F("End time: %s\n") % end_time.to_iso8601_in_utc(); + output << F("Duration: %ss\n") % junit_duration(end_time - start_time); + return output.str(); +} + + +/// Constructor for the hooks. +/// +/// \param [out] output_ Stream to which to write the report. +drivers::report_junit_hooks::report_junit_hooks(std::ostream& output_) : + _output(output_) +{ +} + + +/// Callback executed when the context is loaded. +/// +/// \param context The context loaded from the database. +void +drivers::report_junit_hooks::got_context(const model::context& context) +{ + _output << "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"; + _output << "<testsuite>\n"; + + _output << "<properties>\n"; + _output << F("<property name=\"cwd\" value=\"%s\"/>\n") + % text::escape_xml(context.cwd().str()); + for (model::properties_map::const_iterator iter = + context.env().begin(); iter != context.env().end(); ++iter) { + _output << F("<property name=\"env.%s\" value=\"%s\"/>\n") + % text::escape_xml((*iter).first) + % text::escape_xml((*iter).second); + } + _output << "</properties>\n"; +} + + +/// Callback executed when a test results is found. +/// +/// \param iter Container for the test result's data. +void +drivers::report_junit_hooks::got_result(store::results_iterator& iter) +{ + const model::test_result result = iter.result(); + + _output << F("<testcase classname=\"%s\" name=\"%s\" time=\"%s\">\n") + % text::escape_xml(junit_classname(*iter.test_program())) + % text::escape_xml(iter.test_case_name()) + % junit_duration(iter.end_time() - iter.start_time()); + + std::string stderr_contents; + + switch (result.type()) { + case model::test_result_failed: + _output << F("<failure message=\"%s\"/>\n") + % text::escape_xml(result.reason()); + break; + + case model::test_result_expected_failure: + stderr_contents += ("Expected failure result details\n" + "-------------------------------\n" + "\n" + + result.reason() + "\n" + "\n"); + break; + + case model::test_result_passed: + // Passed results have no status nodes. + break; + + case model::test_result_skipped: + _output << "<skipped/>\n"; + stderr_contents += ("Skipped result details\n" + "----------------------\n" + "\n" + + result.reason() + "\n" + "\n"); + break; + + default: + _output << F("<error message=\"%s\"/>\n") + % text::escape_xml(result.reason()); + } + + const std::string stdout_contents = iter.stdout_contents(); + if (!stdout_contents.empty()) { + _output << F("<system-out>%s</system-out>\n") + % text::escape_xml(stdout_contents); + } + + { + const model::test_case& test_case = iter.test_program()->find( + iter.test_case_name()); + stderr_contents += junit_metadata(test_case.get_metadata()); + } + stderr_contents += junit_timing(iter.start_time(), iter.end_time()); + { + stderr_contents += junit_stderr_header; + const std::string real_stderr_contents = iter.stderr_contents(); + if (real_stderr_contents.empty()) { + stderr_contents += "<EMPTY>\n"; + } else { + stderr_contents += real_stderr_contents; + } + } + _output << "<system-err>" << text::escape_xml(stderr_contents) + << "</system-err>\n"; + + _output << "</testcase>\n"; +} + + +/// Finalizes the report. +void +drivers::report_junit_hooks::end(const drivers::scan_results::result& /* r */) +{ + _output << "</testsuite>\n"; +} diff --git a/drivers/report_junit.hpp b/drivers/report_junit.hpp new file mode 100644 index 000000000000..adb0aa12757e --- /dev/null +++ b/drivers/report_junit.hpp @@ -0,0 +1,75 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file drivers/report_junit.hpp +/// Generates a JUnit report out of a test suite execution. + +#if !defined(ENGINE_REPORT_JUNIT_HPP) +#define ENGINE_REPORT_JUNIT_HPP + +#include <ostream> +#include <string> + +#include "drivers/scan_results.hpp" +#include "model/metadata_fwd.hpp" +#include "model/test_program_fwd.hpp" +#include "utils/datetime_fwd.hpp" + +namespace drivers { + + +extern const char* const junit_metadata_header; +extern const char* const junit_timing_header; +extern const char* const junit_stderr_header; + + +std::string junit_classname(const model::test_program&); +std::string junit_duration(const utils::datetime::delta&); +std::string junit_metadata(const model::metadata&); +std::string junit_timing(const utils::datetime::timestamp&, + const utils::datetime::timestamp&); + + +/// Hooks for the scan_results driver to generate a JUnit report. +class report_junit_hooks : public drivers::scan_results::base_hooks { + /// Stream to which to write the report. + std::ostream& _output; + +public: + report_junit_hooks(std::ostream&); + + void got_context(const model::context&); + void got_result(store::results_iterator&); + + void end(const drivers::scan_results::result&); +}; + + +} // namespace drivers + +#endif // !defined(ENGINE_REPORT_JUNIT_HPP) diff --git a/drivers/report_junit_test.cpp b/drivers/report_junit_test.cpp new file mode 100644 index 000000000000..462dca72f9be --- /dev/null +++ b/drivers/report_junit_test.cpp @@ -0,0 +1,415 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "drivers/report_junit.hpp" + +#include <sstream> +#include <vector> + +#include <atf-c++.hpp> + +#include "drivers/scan_results.hpp" +#include "engine/filters.hpp" +#include "model/context.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "store/write_backend.hpp" +#include "store/write_transaction.hpp" +#include "utils/datetime.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/units.hpp" + +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace units = utils::units; + +using utils::none; + + +namespace { + + +/// Formatted metadata for a test case with defaults. +static const char* const default_metadata = + "allowed_architectures is empty\n" + "allowed_platforms is empty\n" + "description is empty\n" + "has_cleanup = false\n" + "is_exclusive = false\n" + "required_configs is empty\n" + "required_disk_space = 0\n" + "required_files is empty\n" + "required_memory = 0\n" + "required_programs is empty\n" + "required_user is empty\n" + "timeout = 300\n"; + + +/// Formatted metadata for a test case constructed with the "with_metadata" flag +/// set to true in add_tests. +static const char* const overriden_metadata = + "allowed_architectures is empty\n" + "allowed_platforms is empty\n" + "description = Textual description\n" + "has_cleanup = false\n" + "is_exclusive = false\n" + "required_configs is empty\n" + "required_disk_space = 0\n" + "required_files is empty\n" + "required_memory = 0\n" + "required_programs is empty\n" + "required_user is empty\n" + "timeout = 5678\n"; + + +/// Populates the context of the given database. +/// +/// \param tx Transaction to use for the writes to the database. +/// \param env_vars Number of environment variables to add to the context. +static void +add_context(store::write_transaction& tx, const std::size_t env_vars) +{ + std::map< std::string, std::string > env; + for (std::size_t i = 0; i < env_vars; i++) + env[F("VAR%s") % i] = F("Value %s") % i; + const model::context context(fs::path("/root"), env); + (void)tx.put_context(context); +} + + +/// Adds a new test program with various test cases to the given database. +/// +/// \param tx Transaction to use for the writes to the database. +/// \param prog Test program name. +/// \param results Collection of results for the added test cases. The size of +/// this vector indicates the number of tests in the test program. +/// \param with_metadata Whether to add metadata overrides to the test cases. +/// \param with_output Whether to add stdout/stderr messages to the test cases. +static void +add_tests(store::write_transaction& tx, + const char* prog, + const std::vector< model::test_result >& results, + const bool with_metadata, const bool with_output) +{ + model::test_program_builder test_program_builder( + "plain", fs::path(prog), fs::path("/root"), "suite"); + + for (std::size_t j = 0; j < results.size(); j++) { + model::metadata_builder builder; + if (with_metadata) { + builder.set_description("Textual description"); + builder.set_timeout(datetime::delta(5678, 0)); + } + test_program_builder.add_test_case(F("t%s") % j, builder.build()); + } + + const model::test_program test_program = test_program_builder.build(); + const int64_t tp_id = tx.put_test_program(test_program); + + for (std::size_t j = 0; j < results.size(); j++) { + const int64_t tc_id = tx.put_test_case(test_program, F("t%s") % j, + tp_id); + const datetime::timestamp start = + datetime::timestamp::from_microseconds(0); + const datetime::timestamp end = + datetime::timestamp::from_microseconds(j * 1000000 + 500000); + tx.put_result(results[j], tc_id, start, end); + + if (with_output) { + atf::utils::create_file("fake-out", F("stdout file %s") % j); + tx.put_test_case_file("__STDOUT__", fs::path("fake-out"), tc_id); + atf::utils::create_file("fake-err", F("stderr file %s") % j); + tx.put_test_case_file("__STDERR__", fs::path("fake-err"), tc_id); + } + } +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(junit_classname); +ATF_TEST_CASE_BODY(junit_classname) +{ + const model::test_program test_program = model::test_program_builder( + "plain", fs::path("dir1/dir2/program"), fs::path("/root"), "suite") + .build(); + + ATF_REQUIRE_EQ("dir1.dir2.program", drivers::junit_classname(test_program)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(junit_duration); +ATF_TEST_CASE_BODY(junit_duration) +{ + ATF_REQUIRE_EQ("0.457", + drivers::junit_duration(datetime::delta(0, 456700))); + ATF_REQUIRE_EQ("3.120", + drivers::junit_duration(datetime::delta(3, 120000))); + ATF_REQUIRE_EQ("5.000", drivers::junit_duration(datetime::delta(5, 0))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(junit_metadata__defaults); +ATF_TEST_CASE_BODY(junit_metadata__defaults) +{ + const model::metadata metadata = model::metadata_builder().build(); + + const std::string expected = std::string() + + drivers::junit_metadata_header + + default_metadata; + + ATF_REQUIRE_EQ(expected, drivers::junit_metadata(metadata)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(junit_metadata__overrides); +ATF_TEST_CASE_BODY(junit_metadata__overrides) +{ + const model::metadata metadata = model::metadata_builder() + .add_allowed_architecture("arch1") + .add_allowed_platform("platform1") + .set_description("This is a test") + .set_has_cleanup(true) + .set_is_exclusive(true) + .add_required_config("config1") + .set_required_disk_space(units::bytes(456)) + .add_required_file(fs::path("file1")) + .set_required_memory(units::bytes(123)) + .add_required_program(fs::path("prog1")) + .set_required_user("root") + .set_timeout(datetime::delta(10, 0)) + .build(); + + const std::string expected = std::string() + + drivers::junit_metadata_header + + "allowed_architectures = arch1\n" + + "allowed_platforms = platform1\n" + + "description = This is a test\n" + + "has_cleanup = true\n" + + "is_exclusive = true\n" + + "required_configs = config1\n" + + "required_disk_space = 456\n" + + "required_files = file1\n" + + "required_memory = 123\n" + + "required_programs = prog1\n" + + "required_user = root\n" + + "timeout = 10\n"; + + ATF_REQUIRE_EQ(expected, drivers::junit_metadata(metadata)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(junit_timing); +ATF_TEST_CASE_BODY(junit_timing) +{ + const std::string expected = std::string() + + drivers::junit_timing_header + + "Start time: 2015-06-12T01:02:35.123456Z\n" + "End time: 2016-07-13T18:47:10.000001Z\n" + "Duration: 34364674.877s\n"; + + const datetime::timestamp start_time = + datetime::timestamp::from_values(2015, 6, 12, 1, 2, 35, 123456); + const datetime::timestamp end_time = + datetime::timestamp::from_values(2016, 7, 13, 18, 47, 10, 1); + + ATF_REQUIRE_EQ(expected, drivers::junit_timing(start_time, end_time)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(report_junit_hooks__minimal); +ATF_TEST_CASE_BODY(report_junit_hooks__minimal) +{ + store::write_backend backend = store::write_backend::open_rw( + fs::path("test.db")); + store::write_transaction tx = backend.start_write(); + add_context(tx, 0); + tx.commit(); + backend.close(); + + std::ostringstream output; + + drivers::report_junit_hooks hooks(output); + drivers::scan_results::drive(fs::path("test.db"), + std::set< engine::test_filter >(), + hooks); + + const char* expected = + "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n" + "<testsuite>\n" + "<properties>\n" + "<property name=\"cwd\" value=\"/root\"/>\n" + "</properties>\n" + "</testsuite>\n"; + ATF_REQUIRE_EQ(expected, output.str()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(report_junit_hooks__some_tests); +ATF_TEST_CASE_BODY(report_junit_hooks__some_tests) +{ + std::vector< model::test_result > results1; + results1.push_back(model::test_result( + model::test_result_broken, "Broken")); + results1.push_back(model::test_result( + model::test_result_expected_failure, "XFail")); + results1.push_back(model::test_result( + model::test_result_failed, "Failed")); + std::vector< model::test_result > results2; + results2.push_back(model::test_result( + model::test_result_passed)); + results2.push_back(model::test_result( + model::test_result_skipped, "Skipped")); + + store::write_backend backend = store::write_backend::open_rw( + fs::path("test.db")); + store::write_transaction tx = backend.start_write(); + add_context(tx, 2); + add_tests(tx, "dir/prog-1", results1, false, false); + add_tests(tx, "dir/sub/prog-2", results2, true, true); + tx.commit(); + backend.close(); + + std::ostringstream output; + + drivers::report_junit_hooks hooks(output); + drivers::scan_results::drive(fs::path("test.db"), + std::set< engine::test_filter >(), + hooks); + + const std::string expected = std::string() + + "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n" + "<testsuite>\n" + "<properties>\n" + "<property name=\"cwd\" value=\"/root\"/>\n" + "<property name=\"env.VAR0\" value=\"Value 0\"/>\n" + "<property name=\"env.VAR1\" value=\"Value 1\"/>\n" + "</properties>\n" + + "<testcase classname=\"dir.prog-1\" name=\"t0\" time=\"0.500\">\n" + "<error message=\"Broken\"/>\n" + "<system-err>" + + drivers::junit_metadata_header + + default_metadata + + drivers::junit_timing_header + + "Start time: 1970-01-01T00:00:00.000000Z\n" + "End time: 1970-01-01T00:00:00.500000Z\n" + "Duration: 0.500s\n" + + drivers::junit_stderr_header + + "<EMPTY>\n" + "</system-err>\n" + "</testcase>\n" + + "<testcase classname=\"dir.prog-1\" name=\"t1\" time=\"1.500\">\n" + "<system-err>" + "Expected failure result details\n" + "-------------------------------\n" + "\n" + "XFail\n" + "\n" + + drivers::junit_metadata_header + + default_metadata + + drivers::junit_timing_header + + "Start time: 1970-01-01T00:00:00.000000Z\n" + "End time: 1970-01-01T00:00:01.500000Z\n" + "Duration: 1.500s\n" + + drivers::junit_stderr_header + + "<EMPTY>\n" + "</system-err>\n" + "</testcase>\n" + + "<testcase classname=\"dir.prog-1\" name=\"t2\" time=\"2.500\">\n" + "<failure message=\"Failed\"/>\n" + "<system-err>" + + drivers::junit_metadata_header + + default_metadata + + drivers::junit_timing_header + + "Start time: 1970-01-01T00:00:00.000000Z\n" + "End time: 1970-01-01T00:00:02.500000Z\n" + "Duration: 2.500s\n" + + drivers::junit_stderr_header + + "<EMPTY>\n" + "</system-err>\n" + "</testcase>\n" + + "<testcase classname=\"dir.sub.prog-2\" name=\"t0\" time=\"0.500\">\n" + "<system-out>stdout file 0</system-out>\n" + "<system-err>" + + drivers::junit_metadata_header + + overriden_metadata + + drivers::junit_timing_header + + "Start time: 1970-01-01T00:00:00.000000Z\n" + "End time: 1970-01-01T00:00:00.500000Z\n" + "Duration: 0.500s\n" + + drivers::junit_stderr_header + + "stderr file 0</system-err>\n" + "</testcase>\n" + + "<testcase classname=\"dir.sub.prog-2\" name=\"t1\" time=\"1.500\">\n" + "<skipped/>\n" + "<system-out>stdout file 1</system-out>\n" + "<system-err>" + "Skipped result details\n" + "----------------------\n" + "\n" + "Skipped\n" + "\n" + + drivers::junit_metadata_header + + overriden_metadata + + drivers::junit_timing_header + + "Start time: 1970-01-01T00:00:00.000000Z\n" + "End time: 1970-01-01T00:00:01.500000Z\n" + "Duration: 1.500s\n" + + drivers::junit_stderr_header + + "stderr file 1</system-err>\n" + "</testcase>\n" + + "</testsuite>\n"; + ATF_REQUIRE_EQ(expected, output.str()); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, junit_classname); + + ATF_ADD_TEST_CASE(tcs, junit_duration); + + ATF_ADD_TEST_CASE(tcs, junit_metadata__defaults); + ATF_ADD_TEST_CASE(tcs, junit_metadata__overrides); + + ATF_ADD_TEST_CASE(tcs, junit_timing); + + ATF_ADD_TEST_CASE(tcs, report_junit_hooks__minimal); + ATF_ADD_TEST_CASE(tcs, report_junit_hooks__some_tests); +} diff --git a/drivers/run_tests.cpp b/drivers/run_tests.cpp new file mode 100644 index 000000000000..d92940005242 --- /dev/null +++ b/drivers/run_tests.cpp @@ -0,0 +1,344 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "drivers/run_tests.hpp" + +#include <utility> + +#include "engine/config.hpp" +#include "engine/filters.hpp" +#include "engine/kyuafile.hpp" +#include "engine/scanner.hpp" +#include "engine/scheduler.hpp" +#include "model/context.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "store/write_backend.hpp" +#include "store/write_transaction.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/defs.hpp" +#include "utils/format/macros.hpp" +#include "utils/logging/macros.hpp" +#include "utils/noncopyable.hpp" +#include "utils/optional.ipp" +#include "utils/passwd.hpp" +#include "utils/text/operations.ipp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace passwd = utils::passwd; +namespace scheduler = engine::scheduler; +namespace text = utils::text; + +using utils::none; +using utils::optional; + + +namespace { + + +/// Map of test program identifiers (relative paths) to their identifiers in the +/// database. We need to keep this in memory because test programs can be +/// returned by the scanner in any order, and we only want to put each test +/// program once. +typedef std::map< fs::path, int64_t > path_to_id_map; + + +/// Map of in-flight PIDs to their corresponding test case IDs. +typedef std::map< int, int64_t > pid_to_id_map; + + +/// Pair of PID to a test case ID. +typedef pid_to_id_map::value_type pid_and_id_pair; + + +/// Puts a test program in the store and returns its identifier. +/// +/// This function is idempotent: we maintain a side cache of already-put test +/// programs so that we can return their identifiers without having to put them +/// again. +/// TODO(jmmv): It's possible that the store module should offer this +/// functionality and not have to do this ourselves here. +/// +/// \param test_program The test program being put. +/// \param [in,out] tx Writable transaction on the store. +/// \param [in,out] ids_cache Cache of already-put test programs. +/// +/// \return A test program identifier. +static int64_t +find_test_program_id(const model::test_program_ptr test_program, + store::write_transaction& tx, + path_to_id_map& ids_cache) +{ + const fs::path& key = test_program->relative_path(); + std::map< fs::path, int64_t >::const_iterator iter = ids_cache.find(key); + if (iter == ids_cache.end()) { + const int64_t id = tx.put_test_program(*test_program); + ids_cache.insert(std::make_pair(key, id)); + return id; + } else { + return (*iter).second; + } +} + + +/// Stores the result of an execution in the database. +/// +/// \param test_case_id Identifier of the test case in the database. +/// \param result The result of the execution. +/// \param [in,out] tx Writable transaction where to store the result data. +static void +put_test_result(const int64_t test_case_id, + const scheduler::test_result_handle& result, + store::write_transaction& tx) +{ + tx.put_result(result.test_result(), test_case_id, + result.start_time(), result.end_time()); + tx.put_test_case_file("__STDOUT__", result.stdout_file(), test_case_id); + tx.put_test_case_file("__STDERR__", result.stderr_file(), test_case_id); + +} + + +/// Cleans up a test case and folds any errors into the test result. +/// +/// \param handle The result handle for the test. +/// +/// \return The test result if the cleanup succeeds; a broken test result +/// otherwise. +model::test_result +safe_cleanup(scheduler::test_result_handle handle) throw() +{ + try { + handle.cleanup(); + return handle.test_result(); + } catch (const std::exception& e) { + return model::test_result( + model::test_result_broken, + F("Failed to clean up test case's work directory %s: %s") % + handle.work_directory() % e.what()); + } +} + + +/// Starts a test asynchronously. +/// +/// \param handle Scheduler handle. +/// \param match Test program and test case to start. +/// \param [in,out] tx Writable transaction to obtain test IDs. +/// \param [in,out] ids_cache Cache of already-put test cases. +/// \param user_config The end-user configuration properties. +/// \param hooks The hooks for this execution. +/// +/// \returns The PID for the started test and the test case's identifier in the +/// store. +pid_and_id_pair +start_test(scheduler::scheduler_handle& handle, + const engine::scan_result& match, + store::write_transaction& tx, + path_to_id_map& ids_cache, + const config::tree& user_config, + drivers::run_tests::base_hooks& hooks) +{ + const model::test_program_ptr test_program = match.first; + const std::string& test_case_name = match.second; + + hooks.got_test_case(*test_program, test_case_name); + + const int64_t test_program_id = find_test_program_id( + test_program, tx, ids_cache); + const int64_t test_case_id = tx.put_test_case( + *test_program, test_case_name, test_program_id); + + const scheduler::exec_handle exec_handle = handle.spawn_test( + test_program, test_case_name, user_config); + return std::make_pair(exec_handle, test_case_id); +} + + +/// Processes the completion of a test. +/// +/// \param [in,out] result_handle The completion handle of the test subprocess. +/// \param test_case_id Identifier of the test case as returned by start_test(). +/// \param [in,out] tx Writable transaction to put the test results. +/// \param hooks The hooks for this execution. +/// +/// \post result_handle is cleaned up. The caller cannot clean it up again. +void +finish_test(scheduler::result_handle_ptr result_handle, + const int64_t test_case_id, + store::write_transaction& tx, + drivers::run_tests::base_hooks& hooks) +{ + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + + put_test_result(test_case_id, *test_result_handle, tx); + + const model::test_result test_result = safe_cleanup(*test_result_handle); + hooks.got_result( + *test_result_handle->test_program(), + test_result_handle->test_case_name(), + test_result_handle->test_result(), + result_handle->end_time() - result_handle->start_time()); +} + + +/// Extracts the keys of a pid_to_id_map and returns them as a string. +/// +/// \param map The PID to test ID map from which to get the PIDs. +/// +/// \return A user-facing string with the collection of PIDs. +static std::string +format_pids(const pid_to_id_map& map) +{ + std::set< pid_to_id_map::key_type > pids; + for (pid_to_id_map::const_iterator iter = map.begin(); iter != map.end(); + ++iter) { + pids.insert(iter->first); + } + return text::join(pids, ","); +} + + +} // anonymous namespace + + +/// Pure abstract destructor. +drivers::run_tests::base_hooks::~base_hooks(void) +{ +} + + +/// Executes the operation. +/// +/// \param kyuafile_path The path to the Kyuafile to be loaded. +/// \param build_root If not none, path to the built test programs. +/// \param store_path The path to the store to be used. +/// \param filters The test case filters as provided by the user. +/// \param user_config The end-user configuration properties. +/// \param hooks The hooks for this execution. +/// +/// \returns A structure with all results computed by this driver. +drivers::run_tests::result +drivers::run_tests::drive(const fs::path& kyuafile_path, + const optional< fs::path > build_root, + const fs::path& store_path, + const std::set< engine::test_filter >& filters, + const config::tree& user_config, + base_hooks& hooks) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + const engine::kyuafile kyuafile = engine::kyuafile::load( + kyuafile_path, build_root, user_config, handle); + store::write_backend db = store::write_backend::open_rw(store_path); + store::write_transaction tx = db.start_write(); + + { + const model::context context = scheduler::current_context(); + (void)tx.put_context(context); + } + + engine::scanner scanner(kyuafile.test_programs(), filters); + + path_to_id_map ids_cache; + pid_to_id_map in_flight; + std::vector< engine::scan_result > exclusive_tests; + + const std::size_t slots = user_config.lookup< config::positive_int_node >( + "parallelism"); + INV(slots >= 1); + do { + INV(in_flight.size() <= slots); + + // Spawn as many jobs as needed to fill our execution slots. We do this + // first with the assumption that the spawning is faster than any single + // job, so we want to keep as many jobs in the background as possible. + while (in_flight.size() < slots) { + optional< engine::scan_result > match = scanner.yield(); + if (!match) + break; + const model::test_program_ptr test_program = match.get().first; + const std::string& test_case_name = match.get().second; + + const model::test_case& test_case = test_program->find( + test_case_name); + if (test_case.get_metadata().is_exclusive()) { + // Exclusive tests get processed later, separately. + exclusive_tests.push_back(match.get()); + continue; + } + + const pid_and_id_pair pid_id = start_test( + handle, match.get(), tx, ids_cache, user_config, hooks); + INV_MSG(in_flight.find(pid_id.first) == in_flight.end(), + F("Spawned test has PID of still-tracked process %s") % + pid_id.first); + in_flight.insert(pid_id); + } + + // If there are any used slots, consume any at random and return the + // result. We consume slots one at a time to give preference to the + // spawning of new tests as detailed above. + if (!in_flight.empty()) { + scheduler::result_handle_ptr result_handle = handle.wait_any(); + + const pid_to_id_map::iterator iter = in_flight.find( + result_handle->original_pid()); + INV_MSG(iter != in_flight.end(), + F("Lost track of in-flight PID %s; tracking %s") % + result_handle->original_pid() % format_pids(in_flight)); + const int64_t test_case_id = (*iter).second; + in_flight.erase(iter); + + finish_test(result_handle, test_case_id, tx, hooks); + } + } while (!in_flight.empty() || !scanner.done()); + + // Run any exclusive tests that we spotted earlier sequentially. + for (std::vector< engine::scan_result >::const_iterator + iter = exclusive_tests.begin(); iter != exclusive_tests.end(); + ++iter) { + const pid_and_id_pair data = start_test( + handle, *iter, tx, ids_cache, user_config, hooks); + scheduler::result_handle_ptr result_handle = handle.wait_any(); + finish_test(result_handle, data.second, tx, hooks); + } + + tx.commit(); + + handle.cleanup(); + + return result(scanner.unused_filters()); +} diff --git a/drivers/run_tests.hpp b/drivers/run_tests.hpp new file mode 100644 index 000000000000..7f09953d4e03 --- /dev/null +++ b/drivers/run_tests.hpp @@ -0,0 +1,106 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file drivers/run_tests.hpp +/// Driver to run a collection of tests. +/// +/// This driver module implements the logic to execute a collection of tests. +/// The presentation layer is able to monitor progress by hooking into +/// particular points of the driver. + +#if !defined(DRIVERS_RUN_TESTS_HPP) +#define DRIVERS_RUN_TESTS_HPP + +#include <set> +#include <string> + +#include "engine/filters.hpp" +#include "model/test_program.hpp" +#include "model/test_result_fwd.hpp" +#include "utils/config/tree_fwd.hpp" +#include "utils/datetime_fwd.hpp" +#include "utils/fs/path_fwd.hpp" +#include "utils/optional_fwd.hpp" + +namespace drivers { +namespace run_tests { + + +/// Abstract definition of the hooks for this driver. +class base_hooks { +public: + virtual ~base_hooks(void) = 0; + + /// Called when the processing of a test case begins. + /// + /// \param test_program The test program containing the test case. + /// \param test_case_name The name of the test case being executed. + virtual void got_test_case(const model::test_program& test_program, + const std::string& test_case_name) = 0; + + /// Called when a result of a test case becomes available. + /// + /// \param test_program The test program containing the test case. + /// \param test_case_name The name of the executed test case. + /// \param result The result of the execution of the test case. + /// \param duration The time it took to run the test. + virtual void got_result(const model::test_program& test_program, + const std::string& test_case_name, + const model::test_result& result, + const utils::datetime::delta& duration) = 0; +}; + + +/// Tuple containing the results of this driver. +class result { +public: + /// Filters that did not match any available test case. + /// + /// The presence of any filters here probably indicates a usage error. If a + /// test filter does not match any test case, it is probably a typo. + std::set< engine::test_filter > unused_filters; + + /// Initializer for the tuple's fields. + /// + /// \param unused_filters_ The filters that did not match any test case. + result(const std::set< engine::test_filter >& unused_filters_) : + unused_filters(unused_filters_) + { + } +}; + + +result drive(const utils::fs::path&, const utils::optional< utils::fs::path >, + const utils::fs::path&, const std::set< engine::test_filter >&, + const utils::config::tree&, base_hooks&); + + +} // namespace run_tests +} // namespace drivers + +#endif // !defined(DRIVERS_RUN_TESTS_HPP) diff --git a/drivers/scan_results.cpp b/drivers/scan_results.cpp new file mode 100644 index 000000000000..e013cd1fb314 --- /dev/null +++ b/drivers/scan_results.cpp @@ -0,0 +1,107 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "drivers/scan_results.hpp" + +#include "engine/filters.hpp" +#include "model/context.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "store/read_backend.hpp" +#include "store/read_transaction.hpp" +#include "utils/defs.hpp" + +namespace fs = utils::fs; + + +/// Pure abstract destructor. +drivers::scan_results::base_hooks::~base_hooks(void) +{ +} + + +/// Callback executed before any operation is performed. +void +drivers::scan_results::base_hooks::begin(void) +{ +} + + +/// Callback executed after all operations are performed. +void +drivers::scan_results::base_hooks::end(const result& /* r */) +{ +} + + +/// Executes the operation. +/// +/// \param store_path The path to the database store. +/// \param raw_filters The test case filters as provided by the user. +/// \param hooks The hooks for this execution. +/// +/// \returns A structure with all results computed by this driver. +drivers::scan_results::result +drivers::scan_results::drive(const fs::path& store_path, + const std::set< engine::test_filter >& raw_filters, + base_hooks& hooks) +{ + engine::filters_state filters(raw_filters); + + store::read_backend db = store::read_backend::open_ro(store_path); + store::read_transaction tx = db.start_read(); + + hooks.begin(); + + const model::context context = tx.get_context(); + hooks.got_context(context); + + store::results_iterator iter = tx.get_results(); + while (iter) { + // TODO(jmmv): We should be filtering at the test case level for + // efficiency, but that means we would need to execute more than one + // query on the database and our current interfaces don't support that. + // + // Reuse engine::filters_state for the time being because it is simpler + // and we get tracking of unmatched filters "for free". + const model::test_program_ptr test_program = iter.test_program(); + if (filters.match_test_program(test_program->relative_path())) { + const model::test_case& test_case = test_program->find( + iter.test_case_name()); + if (filters.match_test_case(test_program->relative_path(), + test_case.name())) { + hooks.got_result(iter); + } + } + ++iter; + } + + result r(filters.unused()); + hooks.end(r); + return r; +} diff --git a/drivers/scan_results.hpp b/drivers/scan_results.hpp new file mode 100644 index 000000000000..ddf099ae3565 --- /dev/null +++ b/drivers/scan_results.hpp @@ -0,0 +1,105 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file drivers/scan_results.hpp +/// Driver to scan the contents of a results file. +/// +/// This driver module implements the logic to scan the contents of a results +/// file and to notify the presentation layer as soon as data becomes +/// available. This is to prevent reading all the data from the file at once, +/// which could take too much memory. + +#if !defined(DRIVERS_SCAN_RESULTS_HPP) +#define DRIVERS_SCAN_RESULTS_HPP + +extern "C" { +#include <stdint.h> +} + +#include <set> + +#include "engine/filters.hpp" +#include "model/context_fwd.hpp" +#include "store/read_transaction_fwd.hpp" +#include "utils/datetime_fwd.hpp" +#include "utils/fs/path_fwd.hpp" + +namespace drivers { +namespace scan_results { + + +/// Tuple containing the results of this driver. +class result { +public: + /// Filters that did not match any available test case. + /// + /// The presence of any filters here probably indicates a usage error. If a + /// test filter does not match any test case, it is probably a typo. + std::set< engine::test_filter > unused_filters; + + /// Initializer for the tuple's fields. + /// + /// \param unused_filters_ The filters that did not match any test case. + result(const std::set< engine::test_filter >& unused_filters_) : + unused_filters(unused_filters_) + { + } +}; + + +/// Abstract definition of the hooks for this driver. +class base_hooks { +public: + virtual ~base_hooks(void) = 0; + + virtual void begin(void); + + /// Callback executed when the context is loaded. + /// + /// \param context The context loaded from the database. + virtual void got_context(const model::context& context) = 0; + + /// Callback executed when a test results is found. + /// + /// \param iter Container for the test result's data. Some of the data are + /// lazily fetched, hence why we receive the object instead of the + /// individual elements. + virtual void got_result(store::results_iterator& iter) = 0; + + virtual void end(const result& r); +}; + + +result drive(const utils::fs::path&, const std::set< engine::test_filter >&, + base_hooks&); + + +} // namespace scan_results +} // namespace drivers + +#endif // !defined(DRIVERS_SCAN_RESULTS_HPP) diff --git a/drivers/scan_results_test.cpp b/drivers/scan_results_test.cpp new file mode 100644 index 000000000000..5519cb670d85 --- /dev/null +++ b/drivers/scan_results_test.cpp @@ -0,0 +1,258 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "drivers/scan_results.hpp" + +#include <set> + +#include <atf-c++.hpp> + +#include "engine/filters.hpp" +#include "model/context.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "store/exceptions.hpp" +#include "store/read_transaction.hpp" +#include "store/write_backend.hpp" +#include "store/write_transaction.hpp" +#include "utils/datetime.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/optional.ipp" +#include "utils/sanity.hpp" + +namespace datetime = utils::datetime; +namespace fs = utils::fs; + +using utils::none; +using utils::optional; + + +namespace { + + +/// Records the callback values for futher investigation. +class capture_hooks : public drivers::scan_results::base_hooks { +public: + /// Whether begin() was called or not. + bool _begin_called; + + /// The captured driver result, if any. + optional< drivers::scan_results::result > _end_result; + + /// The captured context, if any. + optional< model::context > _context; + + /// The captured results, flattened as "program:test_case:result". + std::set< std::string > _results; + + /// Constructor. + capture_hooks(void) : + _begin_called(false) + { + } + + /// Callback executed before any operation is performed. + void + begin(void) + { + _begin_called = true; + } + + /// Callback executed after all operations are performed. + /// + /// \param r A structure with all results computed by this driver. Note + /// that this is also returned by the drive operation. + void + end(const drivers::scan_results::result& r) + { + PRE(!_end_result); + _end_result = r; + } + + /// Callback executed when the context is loaded. + /// + /// \param context The context loaded from the database. + void got_context(const model::context& context) + { + PRE(!_context); + _context = context; + } + + /// Callback executed when a test results is found. + /// + /// \param iter Container for the test result's data. + void got_result(store::results_iterator& iter) + { + const char* type; + switch (iter.result().type()) { + case model::test_result_passed: type = "passed"; break; + case model::test_result_skipped: type = "skipped"; break; + default: + UNREACHABLE_MSG("Formatting unimplemented"); + } + const datetime::delta duration = iter.end_time() - iter.start_time(); + _results.insert(F("%s:%s:%s:%s:%s:%s") % + iter.test_program()->absolute_path() % + iter.test_case_name() % type % iter.result().reason() % + duration.seconds % duration.useconds); + } +}; + + +/// Populates a results file. +/// +/// It is not OK to call this function multiple times on the same file. +/// +/// \param db_name The database to update. +/// \param count The number of "elements" to insert into the results file. +/// Determines the number of test programs and the number of test cases +/// each has. Can be used to determine from the caller which particular +/// results file has been loaded. +static void +populate_results_file(const char* db_name, const int count) +{ + store::write_backend backend = store::write_backend::open_rw( + fs::path(db_name)); + + store::write_transaction tx = backend.start_write(); + + std::map< std::string, std::string > env; + for (int i = 0; i < count; i++) + env[F("VAR%s") % i] = F("Value %s") % i; + const model::context context(fs::path("/root"), env); + tx.put_context(context); + + for (int i = 0; i < count; i++) { + model::test_program_builder test_program_builder( + "fake", fs::path(F("dir/prog_%s") % i), fs::path("/root"), + F("suite_%s") % i); + for (int j = 0; j < count; j++) { + test_program_builder.add_test_case(F("case_%s") % j); + } + const model::test_program test_program = test_program_builder.build(); + const int64_t tp_id = tx.put_test_program(test_program); + + for (int j = 0; j < count; j++) { + const model::test_result result(model::test_result_skipped, + F("Count %s") % j); + const int64_t tc_id = tx.put_test_case(test_program, + F("case_%s") % j, tp_id); + const datetime::timestamp start = + datetime::timestamp::from_microseconds(1000010); + const datetime::timestamp end = + datetime::timestamp::from_microseconds(5000020 + i + j); + tx.put_result(result, tc_id, start, end); + } + } + + tx.commit(); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(ok__all); +ATF_TEST_CASE_BODY(ok__all) +{ + populate_results_file("test.db", 2); + + capture_hooks hooks; + const drivers::scan_results::result result = drivers::scan_results::drive( + fs::path("test.db"), std::set< engine::test_filter >(), hooks); + ATF_REQUIRE(result.unused_filters.empty()); + ATF_REQUIRE(hooks._begin_called); + ATF_REQUIRE(hooks._end_result); + + std::map< std::string, std::string > env; + env["VAR0"] = "Value 0"; + env["VAR1"] = "Value 1"; + const model::context context(fs::path("/root"), env); + ATF_REQUIRE(context == hooks._context.get()); + + std::set< std::string > results; + results.insert("/root/dir/prog_0:case_0:skipped:Count 0:4:10"); + results.insert("/root/dir/prog_0:case_1:skipped:Count 1:4:11"); + results.insert("/root/dir/prog_1:case_0:skipped:Count 0:4:11"); + results.insert("/root/dir/prog_1:case_1:skipped:Count 1:4:12"); + ATF_REQUIRE_EQ(results, hooks._results); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(ok__filters); +ATF_TEST_CASE_BODY(ok__filters) +{ + populate_results_file("test.db", 3); + + std::set< engine::test_filter > filters; + filters.insert(engine::test_filter(fs::path("dir/prog_1"), "")); + filters.insert(engine::test_filter(fs::path("dir/prog_1"), "no")); + filters.insert(engine::test_filter(fs::path("dir/prog_2"), "case_1")); + filters.insert(engine::test_filter(fs::path("dir/prog_3"), "")); + + capture_hooks hooks; + const drivers::scan_results::result result = drivers::scan_results::drive( + fs::path("test.db"), filters, hooks); + ATF_REQUIRE(hooks._begin_called); + ATF_REQUIRE(hooks._end_result); + + std::set< engine::test_filter > unused_filters; + unused_filters.insert(engine::test_filter(fs::path("dir/prog_1"), "no")); + unused_filters.insert(engine::test_filter(fs::path("dir/prog_3"), "")); + ATF_REQUIRE_EQ(unused_filters, result.unused_filters); + + std::set< std::string > results; + results.insert("/root/dir/prog_1:case_0:skipped:Count 0:4:11"); + results.insert("/root/dir/prog_1:case_1:skipped:Count 1:4:12"); + results.insert("/root/dir/prog_1:case_2:skipped:Count 2:4:13"); + results.insert("/root/dir/prog_2:case_1:skipped:Count 1:4:13"); + ATF_REQUIRE_EQ(results, hooks._results); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(missing_db); +ATF_TEST_CASE_BODY(missing_db) +{ + capture_hooks hooks; + ATF_REQUIRE_THROW( + store::error, + drivers::scan_results::drive(fs::path("test.db"), + std::set< engine::test_filter >(), + hooks)); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, ok__all); + ATF_ADD_TEST_CASE(tcs, ok__filters); + ATF_ADD_TEST_CASE(tcs, missing_db); +} |