1 // Copyright 2015 The Kyua Authors.
2 // All rights reserved.
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #include "engine/tap.hpp"
37 #include "engine/exceptions.hpp"
38 #include "engine/tap_parser.hpp"
39 #include "model/test_case.hpp"
40 #include "model/test_program.hpp"
41 #include "model/test_result.hpp"
42 #include "utils/defs.hpp"
43 #include "utils/env.hpp"
44 #include "utils/format/macros.hpp"
45 #include "utils/optional.ipp"
46 #include "utils/process/operations.hpp"
47 #include "utils/process/status.hpp"
48 #include "utils/sanity.hpp"
50 namespace config = utils::config;
51 namespace fs = utils::fs;
52 namespace process = utils::process;
54 using utils::optional;
60 /// Computes the result of a TAP test program termination.
62 /// Timeouts and bad TAP data must be handled by the caller. Here we assume
63 /// that we have been able to successfully parse the TAP output.
65 /// \param summary Parsed TAP data for the test program.
66 /// \param status Exit status of the test program.
68 /// \return A test result.
69 static model::test_result
70 tap_to_result(const engine::tap_summary& summary,
71 const process::status& status)
73 if (summary.bailed_out()) {
74 return model::test_result(model::test_result_failed, "Bailed out");
77 if (summary.plan() == engine::all_skipped_plan) {
78 return model::test_result(model::test_result_skipped,
79 summary.all_skipped_reason());
82 if (summary.not_ok_count() == 0) {
83 if (status.exitstatus() == EXIT_SUCCESS) {
84 return model::test_result(model::test_result_passed);
86 return model::test_result(
87 model::test_result_broken,
88 F("Dubious test program: reported all tests as passed "
89 "but returned exit code %s") % status.exitstatus());
92 const std::size_t total = summary.ok_count() + summary.not_ok_count();
93 return model::test_result(model::test_result_failed,
94 F("%s of %s tests failed") %
95 summary.not_ok_count() % total);
100 } // anonymous namespace
103 /// Executes a test program's list operation.
105 /// This method is intended to be called within a subprocess and is expected
106 /// to terminate execution either by exec(2)ing the test program or by
107 /// exiting with a failure.
109 engine::tap_interface::exec_list(
110 const model::test_program& /* test_program */,
111 const config::properties_map& /* vars */) const
113 ::_exit(EXIT_SUCCESS);
117 /// Computes the test cases list of a test program.
119 /// \return A list of test cases.
120 model::test_cases_map
121 engine::tap_interface::parse_list(
122 const optional< process::status >& /* status */,
123 const fs::path& /* stdout_path */,
124 const fs::path& /* stderr_path */) const
126 return model::test_cases_map_builder().add("main").build();
130 /// Executes a test case of the test program.
132 /// This method is intended to be called within a subprocess and is expected
133 /// to terminate execution either by exec(2)ing the test program or by
134 /// exiting with a failure.
136 /// \param test_program The test program to execute.
137 /// \param test_case_name Name of the test case to invoke.
138 /// \param vars User-provided variables to pass to the test program.
140 engine::tap_interface::exec_test(
141 const model::test_program& test_program,
142 const std::string& test_case_name,
143 const config::properties_map& vars,
144 const fs::path& /* control_directory */) const
146 PRE(test_case_name == "main");
148 for (config::properties_map::const_iterator iter = vars.begin();
149 iter != vars.end(); ++iter) {
150 utils::setenv(F("TEST_ENV_%s") % (*iter).first, (*iter).second);
153 process::args_vector args;
154 process::exec(test_program.absolute_path(), args);
158 /// Computes the result of a test case based on its termination status.
160 /// \param status The termination status of the subprocess used to execute
161 /// the exec_test() method or none if the test timed out.
162 /// \param stdout_path Path to the file containing the stdout of the test.
164 /// \return A test result.
166 engine::tap_interface::compute_result(
167 const optional< process::status >& status,
168 const fs::path& /* control_directory */,
169 const fs::path& stdout_path,
170 const fs::path& /* stderr_path */) const
173 return model::test_result(model::test_result_broken,
174 "Test case timed out");
176 if (status.get().signaled()) {
177 return model::test_result(
178 model::test_result_broken,
179 F("Received signal %s") % status.get().termsig());
182 const tap_summary summary = parse_tap_output(stdout_path);
183 return tap_to_result(summary, status.get());
184 } catch (const load_error& e) {
185 return model::test_result(
186 model::test_result_broken,
187 F("TAP test program yielded invalid data: %s") % e.what());