1 // Copyright 2010 The Kyua Authors.
2 // All rights reserved.
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 #include "model/test_program.hpp"
40 #include <atf-c++.hpp>
42 #include "model/exceptions.hpp"
43 #include "model/metadata.hpp"
44 #include "model/test_case.hpp"
45 #include "model/test_result.hpp"
46 #include "utils/env.hpp"
47 #include "utils/format/containers.ipp"
48 #include "utils/format/macros.hpp"
49 #include "utils/fs/operations.hpp"
50 #include "utils/fs/path.hpp"
51 #include "utils/optional.ipp"
53 namespace fs = utils::fs;
59 /// Test program that sets its test cases lazily.
61 /// This test class exists to test the behavior of a test_program object when
62 /// the class is extended to offer lazy loading of test cases. We simulate such
63 /// lazy loading here by storing the list of test cases aside at construction
64 /// time and later setting it lazily the first time test_cases() is called.
65 class lazy_test_program : public model::test_program {
66 /// Whether set_test_cases() has yet been called or not.
67 mutable bool _set_test_cases_called;
69 /// The list of test cases for this test program.
71 /// Only use this in the call to set_test_cases(). All other reads of the
72 /// test cases list should happen via the parent class' test_cases() method.
73 model::test_cases_map _lazy_test_cases;
76 /// Constructs a new test program.
78 /// \param interface_name_ Name of the test program interface.
79 /// \param binary_ The name of the test program binary relative to root_.
80 /// \param root_ The root of the test suite containing the test program.
81 /// \param test_suite_name_ The name of the test suite.
82 /// \param metadata_ Metadata of the test program.
83 /// \param test_cases_ The collection of test cases in the test program.
84 lazy_test_program(const std::string& interface_name_,
85 const utils::fs::path& binary_,
86 const utils::fs::path& root_,
87 const std::string& test_suite_name_,
88 const model::metadata& metadata_,
89 const model::test_cases_map& test_cases_) :
90 test_program(interface_name_, binary_, root_, test_suite_name_,
91 metadata_, model::test_cases_map()),
92 _set_test_cases_called(false),
93 _lazy_test_cases(test_cases_)
97 /// Lazily sets the test cases on the parent and returns them.
99 /// \return The list of test cases.
100 const model::test_cases_map&
101 test_cases(void) const
103 if (!_set_test_cases_called) {
104 const_cast< lazy_test_program* >(this)->set_test_cases(
106 _set_test_cases_called = true;
108 return test_program::test_cases();
113 } // anonymous namespace
116 /// Runs a ctor_and_getters test.
118 /// \tparam TestProgram Either model::test_program or lazy_test_program.
119 template< class TestProgram >
121 check_ctor_and_getters(void)
123 const model::metadata tp_md = model::metadata_builder()
124 .add_custom("first", "foo")
125 .add_custom("second", "bar")
127 const model::metadata tc_md = model::metadata_builder()
128 .add_custom("first", "baz")
131 const TestProgram test_program(
132 "mock", fs::path("binary"), fs::path("root"), "suite-name", tp_md,
133 model::test_cases_map_builder().add("foo", tc_md).build());
136 ATF_REQUIRE_EQ("mock", test_program.interface_name());
137 ATF_REQUIRE_EQ(fs::path("binary"), test_program.relative_path());
138 ATF_REQUIRE_EQ(fs::current_path() / "root/binary",
139 test_program.absolute_path());
140 ATF_REQUIRE_EQ(fs::path("root"), test_program.root());
141 ATF_REQUIRE_EQ("suite-name", test_program.test_suite_name());
142 ATF_REQUIRE_EQ(tp_md, test_program.get_metadata());
144 const model::metadata exp_tc_md = model::metadata_builder()
145 .add_custom("first", "baz")
146 .add_custom("second", "bar")
148 const model::test_cases_map exp_tcs = model::test_cases_map_builder()
149 .add("foo", exp_tc_md)
151 ATF_REQUIRE_EQ(exp_tcs, test_program.test_cases());
155 ATF_TEST_CASE_WITHOUT_HEAD(ctor_and_getters);
156 ATF_TEST_CASE_BODY(ctor_and_getters)
158 check_ctor_and_getters< model::test_program >();
162 ATF_TEST_CASE_WITHOUT_HEAD(derived__ctor_and_getters);
163 ATF_TEST_CASE_BODY(derived__ctor_and_getters)
165 check_ctor_and_getters< lazy_test_program >();
169 /// Runs a find_ok test.
171 /// \tparam TestProgram Either model::test_program or lazy_test_program.
172 template< class TestProgram >
176 const model::test_case test_case("main", model::metadata_builder().build());
178 const TestProgram test_program(
179 "mock", fs::path("non-existent"), fs::path("."), "suite-name",
180 model::metadata_builder().build(),
181 model::test_cases_map_builder().add(test_case).build());
183 const model::test_case& found_test_case = test_program.find("main");
184 ATF_REQUIRE_EQ(test_case, found_test_case);
188 ATF_TEST_CASE_WITHOUT_HEAD(find__ok);
189 ATF_TEST_CASE_BODY(find__ok)
191 check_find_ok< model::test_program >();
195 ATF_TEST_CASE_WITHOUT_HEAD(derived__find__ok);
196 ATF_TEST_CASE_BODY(derived__find__ok)
198 check_find_ok< lazy_test_program >();
202 /// Runs a find_missing test.
204 /// \tparam TestProgram Either model::test_program or lazy_test_program.
205 template< class TestProgram >
207 check_find_missing(void)
209 const TestProgram test_program(
210 "mock", fs::path("non-existent"), fs::path("."), "suite-name",
211 model::metadata_builder().build(),
212 model::test_cases_map_builder().add("main").build());
214 ATF_REQUIRE_THROW_RE(model::not_found_error,
215 "case.*abc.*program.*non-existent",
216 test_program.find("abc"));
220 ATF_TEST_CASE_WITHOUT_HEAD(find__missing);
221 ATF_TEST_CASE_BODY(find__missing)
223 check_find_missing< model::test_program >();
227 ATF_TEST_CASE_WITHOUT_HEAD(derived__find__missing);
228 ATF_TEST_CASE_BODY(derived__find__missing)
230 check_find_missing< lazy_test_program >();
234 /// Runs a metadata_inheritance test.
236 /// \tparam TestProgram Either model::test_program or lazy_test_program.
237 template< class TestProgram >
239 check_metadata_inheritance(void)
241 const model::test_cases_map test_cases = model::test_cases_map_builder()
244 model::metadata_builder()
245 .set_description("Overriden description")
248 model::metadata_builder()
249 .add_allowed_architecture("overriden-arch")
250 .add_allowed_platform("overriden-platform")
251 .set_description("Overriden description")
255 const model::metadata metadata = model::metadata_builder()
256 .add_allowed_architecture("base-arch")
257 .set_description("Base description")
259 const TestProgram test_program(
260 "plain", fs::path("non-existent"), fs::path("."), "suite-name",
261 metadata, test_cases);
264 const model::metadata exp_metadata = model::metadata_builder()
265 .add_allowed_architecture("base-arch")
266 .set_description("Base description")
268 ATF_REQUIRE_EQ(exp_metadata,
269 test_program.find("inherit-all").get_metadata());
273 const model::metadata exp_metadata = model::metadata_builder()
274 .add_allowed_architecture("base-arch")
275 .set_description("Overriden description")
277 ATF_REQUIRE_EQ(exp_metadata,
278 test_program.find("inherit-some").get_metadata());
282 const model::metadata exp_metadata = model::metadata_builder()
283 .add_allowed_architecture("overriden-arch")
284 .add_allowed_platform("overriden-platform")
285 .set_description("Overriden description")
287 ATF_REQUIRE_EQ(exp_metadata,
288 test_program.find("inherit-none").get_metadata());
293 ATF_TEST_CASE_WITHOUT_HEAD(metadata_inheritance);
294 ATF_TEST_CASE_BODY(metadata_inheritance)
296 check_metadata_inheritance< model::test_program >();
300 ATF_TEST_CASE_WITHOUT_HEAD(derived__metadata_inheritance);
301 ATF_TEST_CASE_BODY(derived__metadata_inheritance)
303 check_metadata_inheritance< lazy_test_program >();
307 /// Runs a operators_eq_and_ne__copy test.
309 /// \tparam TestProgram Either model::test_program or lazy_test_program.
310 template< class TestProgram >
312 check_operators_eq_and_ne__copy(void)
314 const TestProgram tp1(
315 "plain", fs::path("non-existent"), fs::path("."), "suite-name",
316 model::metadata_builder().build(),
317 model::test_cases_map());
318 const TestProgram tp2 = tp1;
319 ATF_REQUIRE( tp1 == tp2);
320 ATF_REQUIRE(!(tp1 != tp2));
324 ATF_TEST_CASE_WITHOUT_HEAD(operators_eq_and_ne__copy);
325 ATF_TEST_CASE_BODY(operators_eq_and_ne__copy)
327 check_operators_eq_and_ne__copy< model::test_program >();
331 ATF_TEST_CASE_WITHOUT_HEAD(derived__operators_eq_and_ne__copy);
332 ATF_TEST_CASE_BODY(derived__operators_eq_and_ne__copy)
334 check_operators_eq_and_ne__copy< lazy_test_program >();
338 /// Runs a operators_eq_and_ne__not_copy test.
340 /// \tparam TestProgram Either model::test_program or lazy_test_program.
341 template< class TestProgram >
343 check_operators_eq_and_ne__not_copy(void)
345 const std::string base_interface("plain");
346 const fs::path base_relative_path("the/test/program");
347 const fs::path base_root("/the/root");
348 const std::string base_test_suite("suite-name");
349 const model::metadata base_metadata = model::metadata_builder()
350 .add_custom("foo", "bar")
353 const model::test_cases_map base_tcs = model::test_cases_map_builder()
354 .add("main", model::metadata_builder()
355 .add_custom("second", "baz")
359 const TestProgram base_tp(
360 base_interface, base_relative_path, base_root, base_test_suite,
361 base_metadata, base_tcs);
363 // Construct with all same values.
365 const model::test_cases_map other_tcs = model::test_cases_map_builder()
366 .add("main", model::metadata_builder()
367 .add_custom("second", "baz")
371 const TestProgram other_tp(
372 base_interface, base_relative_path, base_root, base_test_suite,
373 base_metadata, other_tcs);
375 ATF_REQUIRE( base_tp == other_tp);
376 ATF_REQUIRE(!(base_tp != other_tp));
379 // Construct with same final metadata values but using a different
380 // intermediate representation. The original test program has one property
381 // in the base test program definition and another in the test case; here,
382 // we put both definitions explicitly in the test case.
384 const model::test_cases_map other_tcs = model::test_cases_map_builder()
385 .add("main", model::metadata_builder()
386 .add_custom("foo", "bar")
387 .add_custom("second", "baz")
391 const TestProgram other_tp(
392 base_interface, base_relative_path, base_root, base_test_suite,
393 base_metadata, other_tcs);
395 ATF_REQUIRE( base_tp == other_tp);
396 ATF_REQUIRE(!(base_tp != other_tp));
399 // Different interface.
401 const TestProgram other_tp(
402 "atf", base_relative_path, base_root, base_test_suite,
403 base_metadata, base_tcs);
405 ATF_REQUIRE(!(base_tp == other_tp));
406 ATF_REQUIRE( base_tp != other_tp);
409 // Different relative path.
411 const TestProgram other_tp(
412 base_interface, fs::path("a/b/c"), base_root, base_test_suite,
413 base_metadata, base_tcs);
415 ATF_REQUIRE(!(base_tp == other_tp));
416 ATF_REQUIRE( base_tp != other_tp);
421 const TestProgram other_tp(
422 base_interface, base_relative_path, fs::path("."), base_test_suite,
423 base_metadata, base_tcs);
425 ATF_REQUIRE(!(base_tp == other_tp));
426 ATF_REQUIRE( base_tp != other_tp);
429 // Different test suite.
431 const TestProgram other_tp(
432 base_interface, base_relative_path, base_root, "different-suite",
433 base_metadata, base_tcs);
435 ATF_REQUIRE(!(base_tp == other_tp));
436 ATF_REQUIRE( base_tp != other_tp);
439 // Different metadata.
441 const TestProgram other_tp(
442 base_interface, base_relative_path, base_root, base_test_suite,
443 model::metadata_builder().build(), base_tcs);
445 ATF_REQUIRE(!(base_tp == other_tp));
446 ATF_REQUIRE( base_tp != other_tp);
449 // Different test cases.
451 const model::test_cases_map other_tcs = model::test_cases_map_builder()
454 const TestProgram other_tp(
455 base_interface, base_relative_path, base_root, base_test_suite,
456 base_metadata, other_tcs);
458 ATF_REQUIRE(!(base_tp == other_tp));
459 ATF_REQUIRE( base_tp != other_tp);
464 ATF_TEST_CASE_WITHOUT_HEAD(operators_eq_and_ne__not_copy);
465 ATF_TEST_CASE_BODY(operators_eq_and_ne__not_copy)
467 check_operators_eq_and_ne__not_copy< model::test_program >();
471 ATF_TEST_CASE_WITHOUT_HEAD(derived__operators_eq_and_ne__not_copy);
472 ATF_TEST_CASE_BODY(derived__operators_eq_and_ne__not_copy)
474 check_operators_eq_and_ne__not_copy< lazy_test_program >();
478 /// Runs a operator_lt test.
480 /// \tparam TestProgram Either model::test_program or lazy_test_program.
481 template< class TestProgram >
483 check_operator_lt(void)
485 const TestProgram tp1(
486 "plain", fs::path("a/b/c"), fs::path("/foo/bar"), "suite-name",
487 model::metadata_builder().build(),
488 model::test_cases_map());
489 const TestProgram tp2(
490 "atf", fs::path("c"), fs::path("/foo/bar"), "suite-name",
491 model::metadata_builder().build(),
492 model::test_cases_map());
493 const TestProgram tp3(
494 "plain", fs::path("a/b/c"), fs::path("/abc"), "suite-name",
495 model::metadata_builder().build(),
496 model::test_cases_map());
498 ATF_REQUIRE(!(tp1 < tp1));
500 ATF_REQUIRE( tp1 < tp2);
501 ATF_REQUIRE(!(tp2 < tp1));
503 ATF_REQUIRE(!(tp1 < tp3));
504 ATF_REQUIRE( tp3 < tp1);
506 // And now, test the actual reason why we want to have an < overload by
507 // attempting to put the various programs in a set.
508 std::set< TestProgram > programs;
509 programs.insert(tp1);
510 programs.insert(tp2);
511 programs.insert(tp3);
515 ATF_TEST_CASE_WITHOUT_HEAD(operator_lt);
516 ATF_TEST_CASE_BODY(operator_lt)
518 check_operator_lt< model::test_program >();
522 ATF_TEST_CASE_WITHOUT_HEAD(derived__operator_lt);
523 ATF_TEST_CASE_BODY(derived__operator_lt)
525 check_operator_lt< lazy_test_program >();
529 /// Runs a output__no_test_cases test.
531 /// \tparam TestProgram Either model::test_program or lazy_test_program.
532 template< class TestProgram >
534 check_output__no_test_cases(void)
537 "plain", fs::path("binary/path"), fs::path("/the/root"), "suite-name",
538 model::metadata_builder().add_allowed_architecture("a").build(),
539 model::test_cases_map());
541 std::ostringstream str;
544 "test_program{interface='plain', binary='binary/path', "
545 "root='/the/root', test_suite='suite-name', "
546 "metadata=metadata{allowed_architectures='a', allowed_platforms='', "
547 "description='', has_cleanup='false', is_exclusive='false', "
548 "required_configs='', required_disk_space='0', required_files='', "
549 "required_memory='0', "
550 "required_programs='', required_user='', timeout='300'}, "
556 ATF_TEST_CASE_WITHOUT_HEAD(output__no_test_cases);
557 ATF_TEST_CASE_BODY(output__no_test_cases)
559 check_output__no_test_cases< model::test_program >();
563 ATF_TEST_CASE_WITHOUT_HEAD(derived__output__no_test_cases);
564 ATF_TEST_CASE_BODY(derived__output__no_test_cases)
566 check_output__no_test_cases< lazy_test_program >();
570 /// Runs a output__some_test_cases test.
572 /// \tparam TestProgram Either model::test_program or lazy_test_program.
573 template< class TestProgram >
575 check_output__some_test_cases(void)
577 const model::test_cases_map test_cases = model::test_cases_map_builder()
578 .add("the-name", model::metadata_builder()
579 .add_allowed_platform("foo")
580 .add_custom("bar", "baz")
585 const TestProgram tp = TestProgram(
586 "plain", fs::path("binary/path"), fs::path("/the/root"), "suite-name",
587 model::metadata_builder().add_allowed_architecture("a").build(),
590 std::ostringstream str;
593 "test_program{interface='plain', binary='binary/path', "
594 "root='/the/root', test_suite='suite-name', "
595 "metadata=metadata{allowed_architectures='a', allowed_platforms='', "
596 "description='', has_cleanup='false', is_exclusive='false', "
597 "required_configs='', required_disk_space='0', required_files='', "
598 "required_memory='0', "
599 "required_programs='', required_user='', timeout='300'}, "
601 "another-name=test_case{name='another-name', "
602 "metadata=metadata{allowed_architectures='a', allowed_platforms='', "
603 "description='', has_cleanup='false', is_exclusive='false', "
604 "required_configs='', required_disk_space='0', required_files='', "
605 "required_memory='0', "
606 "required_programs='', required_user='', timeout='300'}}, "
607 "the-name=test_case{name='the-name', "
608 "metadata=metadata{allowed_architectures='a', allowed_platforms='foo', "
609 "custom.bar='baz', description='', has_cleanup='false', "
610 "is_exclusive='false', "
611 "required_configs='', required_disk_space='0', required_files='', "
612 "required_memory='0', "
613 "required_programs='', required_user='', timeout='300'}})}",
618 ATF_TEST_CASE_WITHOUT_HEAD(output__some_test_cases);
619 ATF_TEST_CASE_BODY(output__some_test_cases)
621 check_output__some_test_cases< model::test_program >();
625 ATF_TEST_CASE_WITHOUT_HEAD(derived__output__some_test_cases);
626 ATF_TEST_CASE_BODY(derived__output__some_test_cases)
628 check_output__some_test_cases< lazy_test_program >();
632 ATF_TEST_CASE_WITHOUT_HEAD(builder__defaults);
633 ATF_TEST_CASE_BODY(builder__defaults)
635 const model::test_program expected(
636 "mock", fs::path("non-existent"), fs::path("."), "suite-name",
637 model::metadata_builder().build(), model::test_cases_map());
639 const model::test_program built = model::test_program_builder(
640 "mock", fs::path("non-existent"), fs::path("."), "suite-name")
643 ATF_REQUIRE_EQ(built, expected);
647 ATF_TEST_CASE_WITHOUT_HEAD(builder__overrides);
648 ATF_TEST_CASE_BODY(builder__overrides)
650 const model::metadata md = model::metadata_builder()
651 .add_custom("foo", "bar")
653 const model::test_cases_map tcs = model::test_cases_map_builder()
657 const model::test_program expected(
658 "mock", fs::path("binary"), fs::path("root"), "suite-name", md, tcs);
660 const model::test_program built = model::test_program_builder(
661 "mock", fs::path("binary"), fs::path("root"), "suite-name")
662 .add_test_case("first")
663 .add_test_case("second", md)
667 ATF_REQUIRE_EQ(built, expected);
671 ATF_TEST_CASE_WITHOUT_HEAD(builder__ptr);
672 ATF_TEST_CASE_BODY(builder__ptr)
674 const model::test_program expected(
675 "mock", fs::path("non-existent"), fs::path("."), "suite-name",
676 model::metadata_builder().build(), model::test_cases_map());
678 const model::test_program_ptr built = model::test_program_builder(
679 "mock", fs::path("non-existent"), fs::path("."), "suite-name")
682 ATF_REQUIRE_EQ(*built, expected);
686 ATF_INIT_TEST_CASES(tcs)
688 ATF_ADD_TEST_CASE(tcs, ctor_and_getters);
689 ATF_ADD_TEST_CASE(tcs, find__ok);
690 ATF_ADD_TEST_CASE(tcs, find__missing);
691 ATF_ADD_TEST_CASE(tcs, metadata_inheritance);
692 ATF_ADD_TEST_CASE(tcs, operators_eq_and_ne__copy);
693 ATF_ADD_TEST_CASE(tcs, operators_eq_and_ne__not_copy);
694 ATF_ADD_TEST_CASE(tcs, operator_lt);
695 ATF_ADD_TEST_CASE(tcs, output__no_test_cases);
696 ATF_ADD_TEST_CASE(tcs, output__some_test_cases);
698 ATF_ADD_TEST_CASE(tcs, derived__ctor_and_getters);
699 ATF_ADD_TEST_CASE(tcs, derived__find__ok);
700 ATF_ADD_TEST_CASE(tcs, derived__find__missing);
701 ATF_ADD_TEST_CASE(tcs, derived__metadata_inheritance);
702 ATF_ADD_TEST_CASE(tcs, derived__operators_eq_and_ne__copy);
703 ATF_ADD_TEST_CASE(tcs, derived__operators_eq_and_ne__not_copy);
704 ATF_ADD_TEST_CASE(tcs, derived__operator_lt);
705 ATF_ADD_TEST_CASE(tcs, derived__output__no_test_cases);
706 ATF_ADD_TEST_CASE(tcs, derived__output__some_test_cases);
708 ATF_ADD_TEST_CASE(tcs, builder__defaults);
709 ATF_ADD_TEST_CASE(tcs, builder__overrides);
710 ATF_ADD_TEST_CASE(tcs, builder__ptr);