1 // Copyright 2015 The Kyua Authors.
2 // All rights reserved.
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #include "engine/tap_parser.hpp"
33 #include <atf-c++.hpp>
35 #include "engine/exceptions.hpp"
36 #include "utils/format/containers.ipp"
37 #include "utils/format/macros.hpp"
38 #include "utils/fs/path.hpp"
40 namespace fs = utils::fs;
46 /// Helper to execute parse_tap_output() on inline text contents.
48 /// \param contents The TAP output to parse.
50 /// \return The tap_summary object resultingafter the parse.
52 /// \throw engine::load_error If parse_tap_output() fails.
53 static engine::tap_summary
54 do_parse(const std::string& contents)
56 std::ofstream output("tap.txt");
60 return engine::parse_tap_output(fs::path("tap.txt"));
64 } // anonymous namespace
67 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__bailed_out);
68 ATF_TEST_CASE_BODY(tap_summary__bailed_out)
70 const engine::tap_summary summary = engine::tap_summary::new_bailed_out();
71 ATF_REQUIRE(summary.bailed_out());
75 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__some_results);
76 ATF_TEST_CASE_BODY(tap_summary__some_results)
78 const engine::tap_summary summary = engine::tap_summary::new_results(
79 engine::tap_plan(1, 5), 3, 2);
80 ATF_REQUIRE(!summary.bailed_out());
81 ATF_REQUIRE_EQ(engine::tap_plan(1, 5), summary.plan());
82 ATF_REQUIRE_EQ(3, summary.ok_count());
83 ATF_REQUIRE_EQ(2, summary.not_ok_count());
87 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__all_skipped);
88 ATF_TEST_CASE_BODY(tap_summary__all_skipped)
90 const engine::tap_summary summary = engine::tap_summary::new_all_skipped(
92 ATF_REQUIRE(!summary.bailed_out());
93 ATF_REQUIRE_EQ(engine::tap_plan(1, 0), summary.plan());
94 ATF_REQUIRE_EQ("Skipped", summary.all_skipped_reason());
98 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__equality_operators);
99 ATF_TEST_CASE_BODY(tap_summary__equality_operators)
101 const engine::tap_summary bailed_out =
102 engine::tap_summary::new_bailed_out();
103 const engine::tap_summary all_skipped_1 =
104 engine::tap_summary::new_all_skipped("Reason 1");
105 const engine::tap_summary results_1 =
106 engine::tap_summary::new_results(engine::tap_plan(1, 5), 3, 2);
108 // Self-equality checks.
109 ATF_REQUIRE( bailed_out == bailed_out);
110 ATF_REQUIRE(!(bailed_out != bailed_out));
111 ATF_REQUIRE( all_skipped_1 == all_skipped_1);
112 ATF_REQUIRE(!(all_skipped_1 != all_skipped_1));
113 ATF_REQUIRE( results_1 == results_1);
114 ATF_REQUIRE(!(results_1 != results_1));
116 // Cross-equality checks.
117 ATF_REQUIRE(!(bailed_out == all_skipped_1));
118 ATF_REQUIRE( bailed_out != all_skipped_1);
119 ATF_REQUIRE(!(bailed_out == results_1));
120 ATF_REQUIRE( bailed_out != results_1);
121 ATF_REQUIRE(!(all_skipped_1 == results_1));
122 ATF_REQUIRE( all_skipped_1 != results_1);
124 // Checks for the all_skipped "type".
125 const engine::tap_summary all_skipped_2 =
126 engine::tap_summary::new_all_skipped("Reason 2");
127 ATF_REQUIRE(!(all_skipped_1 == all_skipped_2));
128 ATF_REQUIRE( all_skipped_1 != all_skipped_2);
131 // Checks for the results "type", different plan.
132 const engine::tap_summary results_2 =
133 engine::tap_summary::new_results(engine::tap_plan(2, 6),
134 results_1.ok_count(),
135 results_1.not_ok_count());
136 ATF_REQUIRE(!(results_1 == results_2));
137 ATF_REQUIRE( results_1 != results_2);
140 // Checks for the results "type", different counts.
141 const engine::tap_summary results_3 =
142 engine::tap_summary::new_results(results_1.plan(),
143 results_1.not_ok_count(),
144 results_1.ok_count());
145 ATF_REQUIRE(!(results_1 == results_3));
146 ATF_REQUIRE( results_1 != results_3);
150 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__output);
151 ATF_TEST_CASE_BODY(tap_summary__output)
154 const engine::tap_summary summary =
155 engine::tap_summary::new_bailed_out();
157 "tap_summary{bailed_out=true}",
158 (F("%s") % summary).str());
162 const engine::tap_summary summary =
163 engine::tap_summary::new_results(engine::tap_plan(5, 10), 2, 4);
165 "tap_summary{bailed_out=false, plan=5..10, ok_count=2, "
167 (F("%s") % summary).str());
171 const engine::tap_summary summary =
172 engine::tap_summary::new_all_skipped("Who knows");
174 "tap_summary{bailed_out=false, plan=1..0, "
175 "all_skipped_reason=Who knows}",
176 (F("%s") % summary).str());
181 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__only_one_result);
182 ATF_TEST_CASE_BODY(parse_tap_output__only_one_result)
184 const engine::tap_summary summary = do_parse(
188 const engine::tap_summary exp_summary =
189 engine::tap_summary::new_results(engine::tap_plan(1, 1), 1, 0);
190 ATF_REQUIRE_EQ(exp_summary, summary);
194 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__all_pass);
195 ATF_TEST_CASE_BODY(parse_tap_output__all_pass)
197 const engine::tap_summary summary = do_parse(
200 " Some diagnostic message\n"
201 "ok - 2 This test also passed\n"
203 "ok - 3 This test passed\n"
204 "not ok 4 # SKIP Some reason\n"
205 "not ok 5 # TODO Another reason\n"
206 "ok - 6 Doesn't make a difference SKIP\n"
207 "ok - 7 Doesn't make a difference either TODO\n"
208 "ok # Also works without a number\n");
210 const engine::tap_summary exp_summary =
211 engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0);
212 ATF_REQUIRE_EQ(exp_summary, summary);
216 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__some_fail);
217 ATF_TEST_CASE_BODY(parse_tap_output__some_fail)
219 const engine::tap_summary summary = do_parse(
221 "not ok - 1 This test failed\n"
222 "ok - 2 This test passed\n"
223 "not ok - 3 This test failed\n"
225 "not ok - 4 This test failed\n"
226 "ok - 5 This test passed\n"
227 "not ok # Fails as well without a number\n");
229 const engine::tap_summary exp_summary =
230 engine::tap_summary::new_results(engine::tap_plan(1, 6), 2, 4);
231 ATF_REQUIRE_EQ(exp_summary, summary);
235 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_and_todo_variants);
236 ATF_TEST_CASE_BODY(parse_tap_output__skip_and_todo_variants)
238 const engine::tap_summary summary = do_parse(
240 "not ok - 1 # SKIP Some reason\n"
241 "not ok - 2 # skip Some reason\n"
242 "not ok - 3 # Skipped Some reason\n"
243 "not ok - 4 # skipped Some reason\n"
244 "not ok - 5 # Skipped: Some reason\n"
245 "not ok - 6 # skipped: Some reason\n"
246 "not ok - 7 # TODO Some reason\n"
247 "not ok - 8 # todo Some reason\n");
249 const engine::tap_summary exp_summary =
250 engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0);
251 ATF_REQUIRE_EQ(exp_summary, summary);
255 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_with_reason);
256 ATF_TEST_CASE_BODY(parse_tap_output__skip_all_with_reason)
258 const engine::tap_summary summary = do_parse(
259 "1..0 SKIP Some reason for skipping\n"
261 " Some diagnostic message\n"
262 "ok - 6 Doesn't make a difference SKIP\n"
263 "ok - 7 Doesn't make a difference either TODO\n");
265 const engine::tap_summary exp_summary =
266 engine::tap_summary::new_all_skipped("Some reason for skipping");
267 ATF_REQUIRE_EQ(exp_summary, summary);
271 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_without_reason);
272 ATF_TEST_CASE_BODY(parse_tap_output__skip_all_without_reason)
274 const engine::tap_summary summary = do_parse(
275 "1..0 unrecognized # garbage skip\n");
277 const engine::tap_summary exp_summary =
278 engine::tap_summary::new_all_skipped("No reason specified");
279 ATF_REQUIRE_EQ(exp_summary, summary);
283 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_invalid);
284 ATF_TEST_CASE_BODY(parse_tap_output__skip_all_invalid)
286 ATF_REQUIRE_THROW_RE(engine::load_error,
287 "Skipped plan must be 1\\.\\.0",
288 do_parse("1..3 # skip\n"));
292 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__plan_at_end);
293 ATF_TEST_CASE_BODY(parse_tap_output__plan_at_end)
295 const engine::tap_summary summary = do_parse(
297 " Some diagnostic message\n"
298 "ok - 2 This test also passed\n"
300 "ok - 3 This test passed\n"
301 "not ok 4 # SKIP Some reason\n"
302 "not ok 5 # TODO Another reason\n"
303 "ok - 6 Doesn't make a difference SKIP\n"
304 "ok - 7 Doesn't make a difference either TODO\n"
307 const engine::tap_summary exp_summary =
308 engine::tap_summary::new_results(engine::tap_plan(1, 7), 7, 0);
309 ATF_REQUIRE_EQ(exp_summary, summary);
313 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__stray_oks);
314 ATF_TEST_CASE_BODY(parse_tap_output__stray_oks)
316 const engine::tap_summary summary = do_parse(
320 "ok - 2 This test also passed\n"
322 "ok - 3 This test passed\n");
324 const engine::tap_summary exp_summary =
325 engine::tap_summary::new_results(engine::tap_plan(1, 3), 3, 0);
326 ATF_REQUIRE_EQ(exp_summary, summary);
330 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__no_plan);
331 ATF_TEST_CASE_BODY(parse_tap_output__no_plan)
333 ATF_REQUIRE_THROW_RE(
335 "Output did not contain any TAP plan",
337 "not ok - 1 This test failed\n"
338 "ok - 2 This test passed\n"));
342 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__double_plan);
343 ATF_TEST_CASE_BODY(parse_tap_output__double_plan)
345 ATF_REQUIRE_THROW_RE(
347 "Found duplicate plan",
351 "not ok - 1 This test failed\n"
352 "ok - 2 This test passed\n"
358 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_plan);
359 ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_plan)
361 ATF_REQUIRE_THROW_RE(
363 "Reported plan differs from actual executed tests",
366 "not ok - 1 This test failed\n"
367 "ok - 2 This test passed\n"));
371 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_trailing_plan);
372 ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_trailing_plan)
374 ATF_REQUIRE_THROW_RE(
376 "Reported plan differs from actual executed tests",
378 "not ok - 1 This test failed\n"
379 "ok - 2 This test passed\n"
384 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__insane_plan);
385 ATF_TEST_CASE_BODY(parse_tap_output__insane_plan)
387 ATF_REQUIRE_THROW_RE(
388 engine::load_error, "Invalid value",
389 do_parse("120830981209831..234891793874080981092803981092312\n"));
393 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__reversed_plan);
394 ATF_TEST_CASE_BODY(parse_tap_output__reversed_plan)
396 ATF_REQUIRE_THROW_RE(engine::load_error,
397 "Found reversed plan 8\\.\\.5",
402 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out);
403 ATF_TEST_CASE_BODY(parse_tap_output__bail_out)
405 const engine::tap_summary summary = do_parse(
407 "not ok - 1 This test failed\n"
408 "Bail out! There is some unknown problem\n"
409 "ok - 2 This test passed\n");
411 const engine::tap_summary exp_summary =
412 engine::tap_summary::new_bailed_out();
413 ATF_REQUIRE_EQ(exp_summary, summary);
417 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out_wins_over_no_plan);
418 ATF_TEST_CASE_BODY(parse_tap_output__bail_out_wins_over_no_plan)
420 const engine::tap_summary summary = do_parse(
421 "not ok - 1 This test failed\n"
422 "Bail out! There is some unknown problem\n"
423 "ok - 2 This test passed\n");
425 const engine::tap_summary exp_summary =
426 engine::tap_summary::new_bailed_out();
427 ATF_REQUIRE_EQ(exp_summary, summary);
431 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__open_failure);
432 ATF_TEST_CASE_BODY(parse_tap_output__open_failure)
434 ATF_REQUIRE_THROW_RE(engine::load_error, "hello.txt.*Failed to open",
435 engine::parse_tap_output(fs::path("hello.txt")));
439 ATF_INIT_TEST_CASES(tcs)
441 ATF_ADD_TEST_CASE(tcs, tap_summary__bailed_out);
442 ATF_ADD_TEST_CASE(tcs, tap_summary__some_results);
443 ATF_ADD_TEST_CASE(tcs, tap_summary__all_skipped);
444 ATF_ADD_TEST_CASE(tcs, tap_summary__equality_operators);
445 ATF_ADD_TEST_CASE(tcs, tap_summary__output);
447 ATF_ADD_TEST_CASE(tcs, parse_tap_output__only_one_result);
448 ATF_ADD_TEST_CASE(tcs, parse_tap_output__all_pass);
449 ATF_ADD_TEST_CASE(tcs, parse_tap_output__some_fail);
450 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_and_todo_variants);
451 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_without_reason);
452 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_with_reason);
453 ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_invalid);
454 ATF_ADD_TEST_CASE(tcs, parse_tap_output__plan_at_end);
455 ATF_ADD_TEST_CASE(tcs, parse_tap_output__stray_oks);
456 ATF_ADD_TEST_CASE(tcs, parse_tap_output__no_plan);
457 ATF_ADD_TEST_CASE(tcs, parse_tap_output__double_plan);
458 ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_plan);
459 ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_trailing_plan);
460 ATF_ADD_TEST_CASE(tcs, parse_tap_output__insane_plan);
461 ATF_ADD_TEST_CASE(tcs, parse_tap_output__reversed_plan);
462 ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out);
463 ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out_wins_over_no_plan);
464 ATF_ADD_TEST_CASE(tcs, parse_tap_output__open_failure);