2 // Automated Testing Framework (atf)
4 // Copyright (c) 2008 The NetBSD Foundation, Inc.
5 // All rights reserved.
7 // Redistribution and use in source and binary forms, with or without
8 // modification, are permitted provided that the following conditions
10 // 1. Redistributions of source code must retain the above copyright
11 // notice, this list of conditions and the following disclaimer.
12 // 2. Redistributions in binary form must reproduce the above copyright
13 // notice, this list of conditions and the following disclaimer in the
14 // documentation and/or other materials provided with the distribution.
16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 #include "detail/fs.hpp"
43 #include "detail/process.hpp"
44 #include "detail/sanity.hpp"
45 #include "detail/test_helpers.hpp"
46 #include "detail/text.hpp"
48 // ------------------------------------------------------------------------
49 // Auxiliary functions.
50 // ------------------------------------------------------------------------
54 create_ctl_file(const char *name)
56 ATF_REQUIRE(open(name, O_CREAT | O_WRONLY | O_TRUNC, 0644) != -1);
59 // ------------------------------------------------------------------------
60 // Auxiliary test cases.
61 // ------------------------------------------------------------------------
63 ATF_TEST_CASE(h_pass);
64 ATF_TEST_CASE_HEAD(h_pass)
66 set_md_var("descr", "Helper test case");
68 ATF_TEST_CASE_BODY(h_pass)
70 create_ctl_file("before");
72 create_ctl_file("after");
75 ATF_TEST_CASE(h_fail);
76 ATF_TEST_CASE_HEAD(h_fail)
78 set_md_var("descr", "Helper test case");
80 ATF_TEST_CASE_BODY(h_fail)
82 create_ctl_file("before");
83 ATF_FAIL("Failed on purpose");
84 create_ctl_file("after");
87 ATF_TEST_CASE(h_skip);
88 ATF_TEST_CASE_HEAD(h_skip)
90 set_md_var("descr", "Helper test case");
92 ATF_TEST_CASE_BODY(h_skip)
94 create_ctl_file("before");
95 ATF_SKIP("Skipped on purpose");
96 create_ctl_file("after");
99 ATF_TEST_CASE(h_require);
100 ATF_TEST_CASE_HEAD(h_require)
102 set_md_var("descr", "Helper test case");
104 ATF_TEST_CASE_BODY(h_require)
106 bool condition = atf::text::to_bool(get_config_var("condition"));
108 create_ctl_file("before");
109 ATF_REQUIRE(condition);
110 create_ctl_file("after");
113 ATF_TEST_CASE(h_require_eq);
114 ATF_TEST_CASE_HEAD(h_require_eq)
116 set_md_var("descr", "Helper test case");
118 ATF_TEST_CASE_BODY(h_require_eq)
120 long v1 = atf::text::to_type< long >(get_config_var("v1"));
121 long v2 = atf::text::to_type< long >(get_config_var("v2"));
123 create_ctl_file("before");
124 ATF_REQUIRE_EQ(v1, v2);
125 create_ctl_file("after");
128 ATF_TEST_CASE(h_require_in);
129 ATF_TEST_CASE_HEAD(h_require_in)
131 set_md_var("descr", "Helper test case");
133 ATF_TEST_CASE_BODY(h_require_in)
135 const std::string element = get_config_var("value");
137 std::set< std::string > collection;
138 collection.insert("foo");
139 collection.insert("bar");
140 collection.insert("baz");
142 create_ctl_file("before");
143 ATF_REQUIRE_IN(element, collection);
144 create_ctl_file("after");
147 ATF_TEST_CASE(h_require_match);
148 ATF_TEST_CASE_HEAD(h_require_match)
150 set_md_var("descr", "Helper test case");
152 ATF_TEST_CASE_BODY(h_require_match)
154 const std::string regexp = get_config_var("regexp");
155 const std::string string = get_config_var("string");
157 create_ctl_file("before");
158 ATF_REQUIRE_MATCH(regexp, string);
159 create_ctl_file("after");
162 ATF_TEST_CASE(h_require_not_in);
163 ATF_TEST_CASE_HEAD(h_require_not_in)
165 set_md_var("descr", "Helper test case");
167 ATF_TEST_CASE_BODY(h_require_not_in)
169 const std::string element = get_config_var("value");
171 std::set< std::string > collection;
172 collection.insert("foo");
173 collection.insert("bar");
174 collection.insert("baz");
176 create_ctl_file("before");
177 ATF_REQUIRE_NOT_IN(element, collection);
178 create_ctl_file("after");
181 ATF_TEST_CASE(h_require_throw);
182 ATF_TEST_CASE_HEAD(h_require_throw)
184 set_md_var("descr", "Helper test case");
186 ATF_TEST_CASE_BODY(h_require_throw)
188 create_ctl_file("before");
190 if (get_config_var("what") == "throw_int")
191 ATF_REQUIRE_THROW(std::runtime_error, if (1) throw int(5));
192 else if (get_config_var("what") == "throw_rt")
193 ATF_REQUIRE_THROW(std::runtime_error,
194 if (1) throw std::runtime_error("e"));
195 else if (get_config_var("what") == "no_throw_rt")
196 ATF_REQUIRE_THROW(std::runtime_error,
197 if (0) throw std::runtime_error("e"));
199 create_ctl_file("after");
202 ATF_TEST_CASE(h_require_throw_re);
203 ATF_TEST_CASE_HEAD(h_require_throw_re)
205 set_md_var("descr", "Helper test case");
207 ATF_TEST_CASE_BODY(h_require_throw_re)
209 create_ctl_file("before");
211 if (get_config_var("what") == "throw_int")
212 ATF_REQUIRE_THROW_RE(std::runtime_error, "5", if (1) throw int(5));
213 else if (get_config_var("what") == "throw_rt_match")
214 ATF_REQUIRE_THROW_RE(std::runtime_error, "foo.*baz",
215 if (1) throw std::runtime_error("a foo bar baz"));
216 else if (get_config_var("what") == "throw_rt_no_match")
217 ATF_REQUIRE_THROW_RE(std::runtime_error, "foo.*baz",
218 if (1) throw std::runtime_error("baz foo bar a"));
219 else if (get_config_var("what") == "no_throw_rt")
220 ATF_REQUIRE_THROW_RE(std::runtime_error, "e",
221 if (0) throw std::runtime_error("e"));
223 create_ctl_file("after");
227 errno_fail_stub(const int raised_errno)
229 errno = raised_errno;
239 ATF_TEST_CASE(h_check_errno);
240 ATF_TEST_CASE_HEAD(h_check_errno)
242 set_md_var("descr", "Helper test case");
244 ATF_TEST_CASE_BODY(h_check_errno)
246 create_ctl_file("before");
248 if (get_config_var("what") == "no_error")
249 ATF_CHECK_ERRNO(-1, errno_ok_stub() == -1);
250 else if (get_config_var("what") == "errno_ok")
251 ATF_CHECK_ERRNO(2, errno_fail_stub(2) == -1);
252 else if (get_config_var("what") == "errno_fail")
253 ATF_CHECK_ERRNO(3, errno_fail_stub(4) == -1);
257 create_ctl_file("after");
260 ATF_TEST_CASE(h_require_errno);
261 ATF_TEST_CASE_HEAD(h_require_errno)
263 set_md_var("descr", "Helper test case");
265 ATF_TEST_CASE_BODY(h_require_errno)
267 create_ctl_file("before");
269 if (get_config_var("what") == "no_error")
270 ATF_REQUIRE_ERRNO(-1, errno_ok_stub() == -1);
271 else if (get_config_var("what") == "errno_ok")
272 ATF_REQUIRE_ERRNO(2, errno_fail_stub(2) == -1);
273 else if (get_config_var("what") == "errno_fail")
274 ATF_REQUIRE_ERRNO(3, errno_fail_stub(4) == -1);
278 create_ctl_file("after");
281 // ------------------------------------------------------------------------
282 // Test cases for the macros.
283 // ------------------------------------------------------------------------
286 ATF_TEST_CASE_HEAD(pass)
288 set_md_var("descr", "Tests the ATF_PASS macro");
290 ATF_TEST_CASE_BODY(pass)
292 ATF_TEST_CASE_USE(h_pass);
293 run_h_tc< ATF_TEST_CASE_NAME(h_pass) >();
294 ATF_REQUIRE(grep_file("result", "^passed"));
295 ATF_REQUIRE(atf::fs::exists(atf::fs::path("before")));
296 ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after")));
300 ATF_TEST_CASE_HEAD(fail)
302 set_md_var("descr", "Tests the ATF_FAIL macro");
304 ATF_TEST_CASE_BODY(fail)
306 ATF_TEST_CASE_USE(h_fail);
307 run_h_tc< ATF_TEST_CASE_NAME(h_fail) >();
308 ATF_REQUIRE(grep_file("result", "^failed: Failed on purpose"));
309 ATF_REQUIRE(atf::fs::exists(atf::fs::path("before")));
310 ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after")));
314 ATF_TEST_CASE_HEAD(skip)
316 set_md_var("descr", "Tests the ATF_SKIP macro");
318 ATF_TEST_CASE_BODY(skip)
320 ATF_TEST_CASE_USE(h_skip);
321 run_h_tc< ATF_TEST_CASE_NAME(h_skip) >();
322 ATF_REQUIRE(grep_file("result", "^skipped: Skipped on purpose"));
323 ATF_REQUIRE(atf::fs::exists(atf::fs::path("before")));
324 ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after")));
327 ATF_TEST_CASE(require);
328 ATF_TEST_CASE_HEAD(require)
330 set_md_var("descr", "Tests the ATF_REQUIRE macro");
332 ATF_TEST_CASE_BODY(require)
343 const atf::fs::path before("before");
344 const atf::fs::path after("after");
346 for (t = &tests[0]; t->cond != NULL; t++) {
347 atf::tests::vars_map config;
348 config["condition"] = t->cond;
350 std::cout << "Checking with a " << t->cond << " value\n";
352 ATF_TEST_CASE_USE(h_require);
353 run_h_tc< ATF_TEST_CASE_NAME(h_require) >(config);
355 ATF_REQUIRE(atf::fs::exists(before));
357 ATF_REQUIRE(grep_file("result", "^passed"));
358 ATF_REQUIRE(atf::fs::exists(after));
360 ATF_REQUIRE(grep_file("result", "^failed: .*condition not met"));
361 ATF_REQUIRE(!atf::fs::exists(after));
364 atf::fs::remove(before);
366 atf::fs::remove(after);
370 ATF_TEST_CASE(require_eq);
371 ATF_TEST_CASE_HEAD(require_eq)
373 set_md_var("descr", "Tests the ATF_REQUIRE_EQ macro");
375 ATF_TEST_CASE_BODY(require_eq)
386 { NULL, NULL, false }
389 const atf::fs::path before("before");
390 const atf::fs::path after("after");
392 for (t = &tests[0]; t->v1 != NULL; t++) {
393 atf::tests::vars_map config;
394 config["v1"] = t->v1;
395 config["v2"] = t->v2;
397 std::cout << "Checking with " << t->v1 << ", " << t->v2
398 << " and expecting " << (t->ok ? "true" : "false")
401 ATF_TEST_CASE_USE(h_require_eq);
402 run_h_tc< ATF_TEST_CASE_NAME(h_require_eq) >(config);
404 ATF_REQUIRE(atf::fs::exists(before));
406 ATF_REQUIRE(grep_file("result", "^passed"));
407 ATF_REQUIRE(atf::fs::exists(after));
409 ATF_REQUIRE(grep_file("result", "^failed: .*v1 != v2"));
410 ATF_REQUIRE(!atf::fs::exists(after));
413 atf::fs::remove(before);
415 atf::fs::remove(after);
419 ATF_TEST_CASE(require_in);
420 ATF_TEST_CASE_HEAD(require_in)
422 set_md_var("descr", "Tests the ATF_REQUIRE_IN macro");
424 ATF_TEST_CASE_BODY(require_in)
439 const atf::fs::path before("before");
440 const atf::fs::path after("after");
442 for (t = &tests[0]; t->value != NULL; t++) {
443 atf::tests::vars_map config;
444 config["value"] = t->value;
446 ATF_TEST_CASE_USE(h_require_in);
447 run_h_tc< ATF_TEST_CASE_NAME(h_require_in) >(config);
449 ATF_REQUIRE(atf::fs::exists(before));
451 ATF_REQUIRE(grep_file("result", "^passed"));
452 ATF_REQUIRE(atf::fs::exists(after));
454 ATF_REQUIRE(grep_file("result", "^failed: "));
455 ATF_REQUIRE(!atf::fs::exists(after));
458 atf::fs::remove(before);
460 atf::fs::remove(after);
464 ATF_TEST_CASE(require_match);
465 ATF_TEST_CASE_HEAD(require_match)
467 set_md_var("descr", "Tests the ATF_REQUIRE_MATCH macro");
469 ATF_TEST_CASE_BODY(require_match)
476 { "foo.*bar", "this is a foo, bar, baz", true },
477 { "bar.*baz", "this is a baz, bar, foo", false },
478 { NULL, NULL, false }
481 const atf::fs::path before("before");
482 const atf::fs::path after("after");
484 for (t = &tests[0]; t->regexp != NULL; t++) {
485 atf::tests::vars_map config;
486 config["regexp"] = t->regexp;
487 config["string"] = t->string;
489 std::cout << "Checking with " << t->regexp << ", " << t->string
490 << " and expecting " << (t->ok ? "true" : "false")
493 ATF_TEST_CASE_USE(h_require_match);
494 run_h_tc< ATF_TEST_CASE_NAME(h_require_match) >(config);
496 ATF_REQUIRE(atf::fs::exists(before));
498 ATF_REQUIRE(grep_file("result", "^passed"));
499 ATF_REQUIRE(atf::fs::exists(after));
501 ATF_REQUIRE(grep_file("result", "^failed: "));
502 ATF_REQUIRE(!atf::fs::exists(after));
505 atf::fs::remove(before);
507 atf::fs::remove(after);
511 ATF_TEST_CASE(require_not_in);
512 ATF_TEST_CASE_HEAD(require_not_in)
514 set_md_var("descr", "Tests the ATF_REQUIRE_NOT_IN macro");
516 ATF_TEST_CASE_BODY(require_not_in)
531 const atf::fs::path before("before");
532 const atf::fs::path after("after");
534 for (t = &tests[0]; t->value != NULL; t++) {
535 atf::tests::vars_map config;
536 config["value"] = t->value;
538 ATF_TEST_CASE_USE(h_require_not_in);
539 run_h_tc< ATF_TEST_CASE_NAME(h_require_not_in) >(config);
541 ATF_REQUIRE(atf::fs::exists(before));
543 ATF_REQUIRE(grep_file("result", "^passed"));
544 ATF_REQUIRE(atf::fs::exists(after));
546 ATF_REQUIRE(grep_file("result", "^failed: "));
547 ATF_REQUIRE(!atf::fs::exists(after));
550 atf::fs::remove(before);
552 atf::fs::remove(after);
556 ATF_TEST_CASE(require_throw);
557 ATF_TEST_CASE_HEAD(require_throw)
559 set_md_var("descr", "Tests the ATF_REQUIRE_THROW macro");
561 ATF_TEST_CASE_BODY(require_throw)
568 { "throw_int", false, "unexpected error" },
569 { "throw_rt", true, NULL },
570 { "no_throw_rt", false, "did not throw" },
571 { NULL, false, NULL }
574 const atf::fs::path before("before");
575 const atf::fs::path after("after");
577 for (t = &tests[0]; t->what != NULL; t++) {
578 atf::tests::vars_map config;
579 config["what"] = t->what;
581 std::cout << "Checking with " << t->what << " and expecting "
582 << (t->ok ? "true" : "false") << "\n";
584 ATF_TEST_CASE_USE(h_require_throw);
585 run_h_tc< ATF_TEST_CASE_NAME(h_require_throw) >(config);
587 ATF_REQUIRE(atf::fs::exists(before));
589 ATF_REQUIRE(grep_file("result", "^passed"));
590 ATF_REQUIRE(atf::fs::exists(after));
592 std::cout << "Checking that message contains '" << t->msg
594 std::string exp_result = std::string("^failed: .*") + t->msg;
595 ATF_REQUIRE(grep_file("result", exp_result.c_str()));
596 ATF_REQUIRE(!atf::fs::exists(after));
599 atf::fs::remove(before);
601 atf::fs::remove(after);
605 ATF_TEST_CASE(require_throw_re);
606 ATF_TEST_CASE_HEAD(require_throw_re)
608 set_md_var("descr", "Tests the ATF_REQUIRE_THROW_RE macro");
610 ATF_TEST_CASE_BODY(require_throw_re)
617 { "throw_int", false, "unexpected error" },
618 { "throw_rt_match", true, NULL },
619 { "throw_rt_no_match", false,
620 "threw.*runtime_error\\(baz foo bar a\\).*"
621 "does not match 'foo\\.\\*baz'" },
622 { "no_throw_rt", false, "did not throw" },
623 { NULL, false, NULL }
626 const atf::fs::path before("before");
627 const atf::fs::path after("after");
629 for (t = &tests[0]; t->what != NULL; t++) {
630 atf::tests::vars_map config;
631 config["what"] = t->what;
633 std::cout << "Checking with " << t->what << " and expecting "
634 << (t->ok ? "true" : "false") << "\n";
636 ATF_TEST_CASE_USE(h_require_throw_re);
637 run_h_tc< ATF_TEST_CASE_NAME(h_require_throw_re) >(config);
639 ATF_REQUIRE(atf::fs::exists(before));
641 ATF_REQUIRE(grep_file("result", "^passed"));
642 ATF_REQUIRE(atf::fs::exists(after));
644 std::cout << "Checking that message contains '" << t->msg
646 std::string exp_result = std::string("^failed: .*") + t->msg;
647 ATF_REQUIRE(grep_file("result", exp_result.c_str()));
648 ATF_REQUIRE(!atf::fs::exists(after));
651 atf::fs::remove(before);
653 atf::fs::remove(after);
657 ATF_TEST_CASE(check_errno);
658 ATF_TEST_CASE_HEAD(check_errno)
660 set_md_var("descr", "Tests the ATF_CHECK_ERRNO macro");
662 ATF_TEST_CASE_BODY(check_errno)
670 "Expected true value in errno_ok_stub\\(\\) == -1" },
671 { "errno_ok", true, NULL },
672 { "errno_fail", false,
673 "Expected errno 3, got 4, in errno_fail_stub\\(4\\) == -1" },
674 { NULL, false, NULL }
677 const atf::fs::path before("before");
678 const atf::fs::path after("after");
680 for (t = &tests[0]; t->what != NULL; t++) {
681 atf::tests::vars_map config;
682 config["what"] = t->what;
684 ATF_TEST_CASE_USE(h_check_errno);
685 run_h_tc< ATF_TEST_CASE_NAME(h_check_errno) >(config);
687 ATF_REQUIRE(atf::fs::exists(before));
688 ATF_REQUIRE(atf::fs::exists(after));
691 ATF_REQUIRE(grep_file("result", "^passed"));
693 ATF_REQUIRE(grep_file("result", "^failed"));
695 std::string exp_result = "macros_test.cpp:[0-9]+: " +
696 std::string(t->msg) + "$";
697 ATF_REQUIRE(grep_file("stderr", exp_result.c_str()));
700 atf::fs::remove(before);
701 atf::fs::remove(after);
705 ATF_TEST_CASE(require_errno);
706 ATF_TEST_CASE_HEAD(require_errno)
708 set_md_var("descr", "Tests the ATF_REQUIRE_ERRNO macro");
710 ATF_TEST_CASE_BODY(require_errno)
718 "Expected true value in errno_ok_stub\\(\\) == -1" },
719 { "errno_ok", true, NULL },
720 { "errno_fail", false,
721 "Expected errno 3, got 4, in errno_fail_stub\\(4\\) == -1" },
722 { NULL, false, NULL }
725 const atf::fs::path before("before");
726 const atf::fs::path after("after");
728 for (t = &tests[0]; t->what != NULL; t++) {
729 atf::tests::vars_map config;
730 config["what"] = t->what;
732 ATF_TEST_CASE_USE(h_require_errno);
733 run_h_tc< ATF_TEST_CASE_NAME(h_require_errno) >(config);
735 ATF_REQUIRE(atf::fs::exists(before));
737 ATF_REQUIRE(grep_file("result", "^passed"));
738 ATF_REQUIRE(atf::fs::exists(after));
740 std::string exp_result = "^failed: .*macros_test.cpp:[0-9]+: " +
741 std::string(t->msg) + "$";
742 ATF_REQUIRE(grep_file("result", exp_result.c_str()));
744 ATF_REQUIRE(!atf::fs::exists(after));
747 atf::fs::remove(before);
749 atf::fs::remove(after);
753 // ------------------------------------------------------------------------
754 // Tests cases for the header file.
755 // ------------------------------------------------------------------------
757 HEADER_TC(include, "atf-c++/macros.hpp");
758 BUILD_TC(use, "macros_hpp_test.cpp",
759 "Tests that the macros provided by the atf-c++/macros.hpp file "
760 "do not cause syntax errors when used",
761 "Build of macros_hpp_test.cpp failed; some macros in "
762 "atf-c++/macros.hpp are broken");
763 BUILD_TC_FAIL(detect_unused_tests, "unused_test.cpp",
764 "Tests that defining an unused test case raises a warning (and thus "
766 "Build of unused_test.cpp passed; unused test cases are not properly "
769 // ------------------------------------------------------------------------
771 // ------------------------------------------------------------------------
773 ATF_INIT_TEST_CASES(tcs)
775 // Add the test cases for the macros.
776 ATF_ADD_TEST_CASE(tcs, pass);
777 ATF_ADD_TEST_CASE(tcs, fail);
778 ATF_ADD_TEST_CASE(tcs, skip);
779 ATF_ADD_TEST_CASE(tcs, check_errno);
780 ATF_ADD_TEST_CASE(tcs, require);
781 ATF_ADD_TEST_CASE(tcs, require_eq);
782 ATF_ADD_TEST_CASE(tcs, require_in);
783 ATF_ADD_TEST_CASE(tcs, require_match);
784 ATF_ADD_TEST_CASE(tcs, require_not_in);
785 ATF_ADD_TEST_CASE(tcs, require_throw);
786 ATF_ADD_TEST_CASE(tcs, require_throw_re);
787 ATF_ADD_TEST_CASE(tcs, require_errno);
789 // Add the test cases for the header file.
790 ATF_ADD_TEST_CASE(tcs, include);
791 ATF_ADD_TEST_CASE(tcs, use);
792 ATF_ADD_TEST_CASE(tcs, detect_unused_tests);