1 // Copyright 2012 The Kyua Authors.
2 // All rights reserved.
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #include "cli/cmd_report_html.hpp"
37 #include "cli/common.ipp"
38 #include "drivers/scan_results.hpp"
39 #include "engine/filters.hpp"
40 #include "model/context.hpp"
41 #include "model/metadata.hpp"
42 #include "model/test_case.hpp"
43 #include "model/test_program.hpp"
44 #include "model/test_result.hpp"
45 #include "store/layout.hpp"
46 #include "store/read_transaction.hpp"
47 #include "utils/cmdline/options.hpp"
48 #include "utils/cmdline/parser.ipp"
49 #include "utils/cmdline/ui.hpp"
50 #include "utils/datetime.hpp"
51 #include "utils/env.hpp"
52 #include "utils/format/macros.hpp"
53 #include "utils/fs/exceptions.hpp"
54 #include "utils/fs/operations.hpp"
55 #include "utils/fs/path.hpp"
56 #include "utils/optional.ipp"
57 #include "utils/text/templates.hpp"
59 namespace cmdline = utils::cmdline;
60 namespace config = utils::config;
61 namespace datetime = utils::datetime;
62 namespace fs = utils::fs;
63 namespace layout = store::layout;
64 namespace text = utils::text;
66 using utils::optional;
72 /// Creates the report's top directory and fails if it exists.
74 /// \param directory The directory to create.
75 /// \param force Whether to wipe an existing directory or not.
77 /// \throw std::runtime_error If the directory already exists; this is a user
78 /// error that the user must correct.
79 /// \throw fs::error If the directory creation fails for any other reason.
81 create_top_directory(const fs::path& directory, const bool force)
84 if (fs::exists(directory))
89 fs::mkdir(directory, 0755);
90 } catch (const fs::system_error& e) {
91 if (e.original_errno() == EEXIST)
92 throw std::runtime_error(F("Output directory '%s' already exists; "
93 "maybe use --force?") %
101 /// Generates a flat unique filename for a given test case.
103 /// \param test_program The test program for which to genereate the name.
104 /// \param test_case_name The test case name.
106 /// \return A filename unique within a directory with a trailing HTML extension.
108 test_case_filename(const model::test_program& test_program,
109 const std::string& test_case_name)
111 static const char* special_characters = "/:";
113 std::string name = cli::format_test_case_id(test_program, test_case_name);
114 std::string::size_type pos = name.find_first_of(special_characters);
115 while (pos != std::string::npos) {
116 name.replace(pos, 1, "_");
117 pos = name.find_first_of(special_characters, pos + 1);
119 return name + ".html";
123 /// Adds a string to string map to the templates.
125 /// \param [in,out] templates The templates to add the map to.
126 /// \param props The map to add to the templates.
127 /// \param key_vector Name of the template vector that holds the keys.
128 /// \param value_vector Name of the template vector that holds the values.
130 add_map(text::templates_def& templates, const config::properties_map& props,
131 const std::string& key_vector, const std::string& value_vector)
133 templates.add_vector(key_vector);
134 templates.add_vector(value_vector);
136 for (config::properties_map::const_iterator iter = props.begin();
137 iter != props.end(); ++iter) {
138 templates.add_to_vector(key_vector, (*iter).first);
139 templates.add_to_vector(value_vector, (*iter).second);
144 /// Generates an HTML report.
145 class html_hooks : public drivers::scan_results::base_hooks {
146 /// User interface object where to report progress.
149 /// The top directory in which to create the HTML files.
152 /// Collection of result types to include in the report.
153 const cli::result_types& _results_filters;
155 /// The start time of the first test.
156 optional< utils::datetime::timestamp > _start_time;
158 /// The end time of the last test.
159 optional< utils::datetime::timestamp > _end_time;
161 /// The total run time of the tests. Note that we cannot subtract _end_time
162 /// from _start_time to compute this due to parallel execution.
163 utils::datetime::delta _runtime;
165 /// Templates accumulator to generate the index.html file.
166 text::templates_def _summary_templates;
168 /// Mapping of result types to the amount of tests with such result.
169 std::map< model::test_result_type, std::size_t > _types_count;
171 /// Generates a common set of templates for all of our files.
173 /// \return A new templates object with common parameters.
174 static text::templates_def
175 common_templates(void)
177 text::templates_def templates;
178 templates.add_variable("css", "report.css");
182 /// Adds a test case result to the summary.
184 /// \param test_program The test program with the test case to be added.
185 /// \param test_case_name Name of the test case.
186 /// \param result The result of the test case.
187 /// \param has_detail If true, the result of the test case has not been
188 /// filtered and therefore there exists a separate file for the test
189 /// with all of its information.
191 add_to_summary(const model::test_program& test_program,
192 const std::string& test_case_name,
193 const model::test_result& result,
194 const bool has_detail)
196 ++_types_count[result.type()];
201 std::string test_cases_vector;
202 std::string test_cases_file_vector;
203 switch (result.type()) {
204 case model::test_result_broken:
205 test_cases_vector = "broken_test_cases";
206 test_cases_file_vector = "broken_test_cases_file";
209 case model::test_result_expected_failure:
210 test_cases_vector = "xfail_test_cases";
211 test_cases_file_vector = "xfail_test_cases_file";
214 case model::test_result_failed:
215 test_cases_vector = "failed_test_cases";
216 test_cases_file_vector = "failed_test_cases_file";
219 case model::test_result_passed:
220 test_cases_vector = "passed_test_cases";
221 test_cases_file_vector = "passed_test_cases_file";
224 case model::test_result_skipped:
225 test_cases_vector = "skipped_test_cases";
226 test_cases_file_vector = "skipped_test_cases_file";
229 INV(!test_cases_vector.empty());
230 INV(!test_cases_file_vector.empty());
232 _summary_templates.add_to_vector(
234 cli::format_test_case_id(test_program, test_case_name));
235 _summary_templates.add_to_vector(
236 test_cases_file_vector,
237 test_case_filename(test_program, test_case_name));
240 /// Instantiate a template to generate an HTML file in the output directory.
242 /// \param templates The templates to use.
243 /// \param template_name The name of the template. This is automatically
244 /// searched for in the installed directory, so do not provide a path.
245 /// \param output_name The name of the output file. This is a basename to
246 /// be created within the output directory.
248 /// \throw text::error If there is any problem applying the templates.
250 generate(const text::templates_def& templates,
251 const std::string& template_name,
252 const std::string& output_name) const
254 const fs::path miscdir(utils::getenv_with_default(
255 "KYUA_MISCDIR", KYUA_MISCDIR));
256 const fs::path template_file = miscdir / template_name;
257 const fs::path output_path(_directory / output_name);
259 _ui->out(F("Generating %s") % output_path);
260 text::instantiate(templates, template_file, output_path);
263 /// Gets the number of tests with a given result type.
265 /// \param type The type to be queried.
267 /// \return The number of tests of the given type, or 0 if none have yet
268 /// been registered by add_to_summary().
270 get_count(const model::test_result_type type) const
272 const std::map< model::test_result_type, std::size_t >::const_iterator
273 iter = _types_count.find(type);
274 if (iter == _types_count.end())
277 return (*iter).second;
281 /// Constructor for the hooks.
283 /// \param ui_ User interface object where to report progress.
284 /// \param directory_ The directory in which to create the HTML files.
285 /// \param results_filters_ The result types to include in the report.
287 html_hooks(cmdline::ui* ui_, const fs::path& directory_,
288 const cli::result_types& results_filters_) :
290 _directory(directory_),
291 _results_filters(results_filters_),
292 _summary_templates(common_templates())
294 PRE(!results_filters_.empty());
296 // Keep in sync with add_to_summary().
297 _summary_templates.add_vector("broken_test_cases");
298 _summary_templates.add_vector("broken_test_cases_file");
299 _summary_templates.add_vector("xfail_test_cases");
300 _summary_templates.add_vector("xfail_test_cases_file");
301 _summary_templates.add_vector("failed_test_cases");
302 _summary_templates.add_vector("failed_test_cases_file");
303 _summary_templates.add_vector("passed_test_cases");
304 _summary_templates.add_vector("passed_test_cases_file");
305 _summary_templates.add_vector("skipped_test_cases");
306 _summary_templates.add_vector("skipped_test_cases_file");
309 /// Callback executed when the context is loaded.
311 /// \param context The context loaded from the database.
313 got_context(const model::context& context)
315 text::templates_def templates = common_templates();
316 templates.add_variable("cwd", context.cwd().str());
317 add_map(templates, context.env(), "env_var", "env_var_value");
318 generate(templates, "context.html", "context.html");
321 /// Callback executed when a test results is found.
323 /// \param iter Container for the test result's data.
325 got_result(store::results_iterator& iter)
327 const model::test_program_ptr test_program = iter.test_program();
328 const std::string& test_case_name = iter.test_case_name();
329 const model::test_result result = iter.result();
331 if (std::find(_results_filters.begin(), _results_filters.end(),
332 result.type()) == _results_filters.end()) {
333 add_to_summary(*test_program, test_case_name, result, false);
337 add_to_summary(*test_program, test_case_name, result, true);
339 if (!_start_time || _start_time.get() > iter.start_time())
340 _start_time = iter.start_time();
341 if (!_end_time || _end_time.get() < iter.end_time())
342 _end_time = iter.end_time();
344 const datetime::delta duration = iter.end_time() - iter.start_time();
346 _runtime += duration;
348 text::templates_def templates = common_templates();
349 templates.add_variable("test_case",
350 cli::format_test_case_id(*test_program,
352 templates.add_variable("test_program",
353 test_program->absolute_path().str());
354 templates.add_variable("result", cli::format_result(result));
355 templates.add_variable("start_time",
356 iter.start_time().to_iso8601_in_utc());
357 templates.add_variable("end_time",
358 iter.end_time().to_iso8601_in_utc());
359 templates.add_variable("duration", cli::format_delta(duration));
361 const model::test_case& test_case = test_program->find(test_case_name);
362 add_map(templates, test_case.get_metadata().to_properties(),
363 "metadata_var", "metadata_value");
366 const std::string stdout_text = iter.stdout_contents();
367 if (!stdout_text.empty())
368 templates.add_variable("stdout", stdout_text);
371 const std::string stderr_text = iter.stderr_contents();
372 if (!stderr_text.empty())
373 templates.add_variable("stderr", stderr_text);
376 generate(templates, "test_result.html",
377 test_case_filename(*test_program, test_case_name));
380 /// Writes the index.html file in the output directory.
382 /// This should only be called once all the processing has been done;
383 /// i.e. when the scan_results driver returns.
387 const std::size_t n_passed = get_count(model::test_result_passed);
388 const std::size_t n_failed = get_count(model::test_result_failed);
389 const std::size_t n_skipped = get_count(model::test_result_skipped);
390 const std::size_t n_xfail = get_count(
391 model::test_result_expected_failure);
392 const std::size_t n_broken = get_count(model::test_result_broken);
394 const std::size_t n_bad = n_broken + n_failed;
398 _summary_templates.add_variable(
399 "start_time", _start_time.get().to_iso8601_in_utc());
400 _summary_templates.add_variable(
401 "end_time", _end_time.get().to_iso8601_in_utc());
403 _summary_templates.add_variable("start_time", "No tests run");
404 _summary_templates.add_variable("end_time", "No tests run");
406 _summary_templates.add_variable("duration",
407 cli::format_delta(_runtime));
408 _summary_templates.add_variable("passed_tests_count",
410 _summary_templates.add_variable("failed_tests_count",
412 _summary_templates.add_variable("skipped_tests_count",
413 F("%s") % n_skipped);
414 _summary_templates.add_variable("xfail_tests_count",
416 _summary_templates.add_variable("broken_tests_count",
418 _summary_templates.add_variable("bad_tests_count", F("%s") % n_bad);
420 generate(text::templates_def(), "report.css", "report.css");
421 generate(_summary_templates, "index.html", "index.html");
426 } // anonymous namespace
429 /// Default constructor for cmd_report_html.
430 cli::cmd_report_html::cmd_report_html(void) : cli_command(
431 "report-html", "", 0, 0,
432 "Generates an HTML report with the result of a test suite run")
434 add_option(results_file_open_option);
435 add_option(cmdline::bool_option(
436 "force", "Wipe the output directory before generating the new report; "
438 add_option(cmdline::path_option(
439 "output", "The directory in which to store the HTML files",
441 add_option(cmdline::list_option(
442 "results-filter", "Comma-separated list of result types to include in "
443 "the report", "types", "skipped,xfail,broken,failed"));
447 /// Entry point for the "report-html" subcommand.
449 /// \param ui Object to interact with the I/O of the program.
450 /// \param cmdline Representation of the command line to the subcommand.
452 /// \return 0 if everything is OK, 1 if the statement is invalid or if there is
453 /// any other problem.
455 cli::cmd_report_html::run(cmdline::ui* ui,
456 const cmdline::parsed_cmdline& cmdline,
457 const config::tree& /* user_config */)
459 const result_types types = get_result_types(cmdline);
461 const fs::path results_file = layout::find_results(
462 results_file_open(cmdline));
464 const fs::path directory =
465 cmdline.get_option< cmdline::path_option >("output");
466 create_top_directory(directory, cmdline.has_option("force"));
467 html_hooks hooks(ui, directory, types);
468 drivers::scan_results::drive(results_file,
469 std::set< engine::test_filter >(),
471 hooks.write_summary();