1 // Copyright 2013 The Kyua Authors.
2 // All rights reserved.
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include <atf-c++.hpp>
33 #include "model/context.hpp"
34 #include "model/metadata.hpp"
35 #include "model/test_program.hpp"
36 #include "model/test_result.hpp"
37 #include "store/migrate.hpp"
38 #include "store/read_backend.hpp"
39 #include "store/read_transaction.hpp"
40 #include "store/write_backend.hpp"
41 #include "utils/datetime.hpp"
42 #include "utils/env.hpp"
43 #include "utils/format/macros.hpp"
44 #include "utils/fs/path.hpp"
45 #include "utils/logging/operations.hpp"
46 #include "utils/sqlite/database.hpp"
47 #include "utils/stream.hpp"
48 #include "utils/units.hpp"
50 namespace datetime = utils::datetime;
51 namespace fs = utils::fs;
52 namespace logging = utils::logging;
53 namespace sqlite = utils::sqlite;
54 namespace units = utils::units;
60 /// Gets a data file from the tests directory.
62 /// We cannot use the srcdir property because the required files are not there
63 /// when building with an object directory. In those cases, the data files
64 /// remainin the source directory while the resulting test program is in the
65 /// object directory, thus having the wrong value for its srcdir property.
67 /// \param name Basename of the test data file to query.
69 /// \return The actual path to the requested data file.
71 testdata_file(const std::string& name)
73 const fs::path testdatadir(utils::getenv_with_default(
74 "KYUA_STORETESTDATADIR", KYUA_STORETESTDATADIR));
75 return testdatadir / name;
79 /// Validates the contents of the action with identifier 1.
81 /// \param dbpath Path to the database in which to check the action contents.
83 check_action_1(const fs::path& dbpath)
85 store::read_backend backend = store::read_backend::open_ro(dbpath);
86 store::read_transaction transaction = backend.start_read();
88 const fs::path root("/some/root");
89 std::map< std::string, std::string > environment;
90 const model::context context(root, environment);
92 ATF_REQUIRE_EQ(context, transaction.get_context());
94 store::results_iterator iter = transaction.get_results();
99 /// Validates the contents of the action with identifier 2.
101 /// \param dbpath Path to the database in which to check the action contents.
103 check_action_2(const fs::path& dbpath)
105 store::read_backend backend = store::read_backend::open_ro(dbpath);
106 store::read_transaction transaction = backend.start_read();
108 const fs::path root("/test/suite/root");
109 std::map< std::string, std::string > environment;
110 environment["HOME"] = "/home/test";
111 environment["PATH"] = "/bin:/usr/bin";
112 const model::context context(root, environment);
114 ATF_REQUIRE_EQ(context, transaction.get_context());
116 const model::test_program test_program_1 = model::test_program_builder(
117 "plain", fs::path("foo_test"), fs::path("/test/suite/root"),
119 .add_test_case("main")
121 const model::test_result result_1(model::test_result_passed);
123 const model::test_program test_program_2 = model::test_program_builder(
124 "plain", fs::path("subdir/another_test"), fs::path("/test/suite/root"),
126 .add_test_case("main",
127 model::metadata_builder()
128 .set_timeout(datetime::delta(10, 0))
130 .set_metadata(model::metadata_builder()
131 .set_timeout(datetime::delta(10, 0))
134 const model::test_result result_2(model::test_result_failed,
135 "Exited with code 1");
137 const model::test_program test_program_3 = model::test_program_builder(
138 "plain", fs::path("subdir/bar_test"), fs::path("/test/suite/root"),
140 .add_test_case("main")
142 const model::test_result result_3(model::test_result_broken,
143 "Received signal 1");
145 const model::test_program test_program_4 = model::test_program_builder(
146 "plain", fs::path("top_test"), fs::path("/test/suite/root"),
148 .add_test_case("main")
150 const model::test_result result_4(model::test_result_expected_failure,
153 const model::test_program test_program_5 = model::test_program_builder(
154 "plain", fs::path("last_test"), fs::path("/test/suite/root"),
156 .add_test_case("main")
158 const model::test_result result_5(model::test_result_skipped,
161 store::results_iterator iter = transaction.get_results();
163 ATF_REQUIRE_EQ(test_program_1, *iter.test_program());
164 ATF_REQUIRE_EQ("main", iter.test_case_name());
165 ATF_REQUIRE_EQ(result_1, iter.result());
166 ATF_REQUIRE(iter.stdout_contents().empty());
167 ATF_REQUIRE(iter.stderr_contents().empty());
168 ATF_REQUIRE_EQ(1357643611000000LL, iter.start_time().to_microseconds());
169 ATF_REQUIRE_EQ(1357643621000500LL, iter.end_time().to_microseconds());
173 ATF_REQUIRE_EQ(test_program_5, *iter.test_program());
174 ATF_REQUIRE_EQ("main", iter.test_case_name());
175 ATF_REQUIRE_EQ(result_5, iter.result());
176 ATF_REQUIRE(iter.stdout_contents().empty());
177 ATF_REQUIRE(iter.stderr_contents().empty());
178 ATF_REQUIRE_EQ(1357643632000000LL, iter.start_time().to_microseconds());
179 ATF_REQUIRE_EQ(1357643638000000LL, iter.end_time().to_microseconds());
183 ATF_REQUIRE_EQ(test_program_2, *iter.test_program());
184 ATF_REQUIRE_EQ("main", iter.test_case_name());
185 ATF_REQUIRE_EQ(result_2, iter.result());
186 ATF_REQUIRE_EQ("Test stdout", iter.stdout_contents());
187 ATF_REQUIRE_EQ("Test stderr", iter.stderr_contents());
188 ATF_REQUIRE_EQ(1357643622001200LL, iter.start_time().to_microseconds());
189 ATF_REQUIRE_EQ(1357643622900021LL, iter.end_time().to_microseconds());
193 ATF_REQUIRE_EQ(test_program_3, *iter.test_program());
194 ATF_REQUIRE_EQ("main", iter.test_case_name());
195 ATF_REQUIRE_EQ(result_3, iter.result());
196 ATF_REQUIRE(iter.stdout_contents().empty());
197 ATF_REQUIRE(iter.stderr_contents().empty());
198 ATF_REQUIRE_EQ(1357643623500000LL, iter.start_time().to_microseconds());
199 ATF_REQUIRE_EQ(1357643630981932LL, iter.end_time().to_microseconds());
203 ATF_REQUIRE_EQ(test_program_4, *iter.test_program());
204 ATF_REQUIRE_EQ("main", iter.test_case_name());
205 ATF_REQUIRE_EQ(result_4, iter.result());
206 ATF_REQUIRE(iter.stdout_contents().empty());
207 ATF_REQUIRE(iter.stderr_contents().empty());
208 ATF_REQUIRE_EQ(1357643631000000LL, iter.start_time().to_microseconds());
209 ATF_REQUIRE_EQ(1357643631020000LL, iter.end_time().to_microseconds());
216 /// Validates the contents of the action with identifier 3.
218 /// \param dbpath Path to the database in which to check the action contents.
220 check_action_3(const fs::path& dbpath)
222 store::read_backend backend = store::read_backend::open_ro(dbpath);
223 store::read_transaction transaction = backend.start_read();
225 const fs::path root("/usr/tests");
226 std::map< std::string, std::string > environment;
227 environment["PATH"] = "/bin:/usr/bin";
228 const model::context context(root, environment);
230 ATF_REQUIRE_EQ(context, transaction.get_context());
232 const model::test_program test_program_6 = model::test_program_builder(
233 "atf", fs::path("complex_test"), fs::path("/usr/tests"),
235 .add_test_case("this_passes")
236 .add_test_case("this_fails",
237 model::metadata_builder()
238 .set_description("Test description")
239 .set_has_cleanup(true)
240 .set_required_memory(units::bytes(128))
241 .set_required_user("root")
243 .add_test_case("this_skips",
244 model::metadata_builder()
245 .add_allowed_architecture("powerpc")
246 .add_allowed_architecture("x86_64")
247 .add_allowed_platform("amd64")
248 .add_allowed_platform("macppc")
249 .add_required_config("X-foo")
250 .add_required_config("unprivileged_user")
251 .add_required_file(fs::path("/the/data/file"))
252 .add_required_program(fs::path("/bin/ls"))
253 .add_required_program(fs::path("cp"))
254 .set_description("Test explanation")
255 .set_has_cleanup(true)
256 .set_required_memory(units::bytes(512))
257 .set_required_user("unprivileged")
258 .set_timeout(datetime::delta(600, 0))
261 const model::test_result result_6(model::test_result_passed);
262 const model::test_result result_7(model::test_result_failed,
264 const model::test_result result_8(model::test_result_skipped,
267 const model::test_program test_program_7 = model::test_program_builder(
268 "atf", fs::path("simple_test"), fs::path("/usr/tests"),
270 .add_test_case("main",
271 model::metadata_builder()
272 .set_description("More text")
273 .set_has_cleanup(true)
274 .set_required_memory(units::bytes(128))
275 .set_required_user("unprivileged")
278 const model::test_result result_9(model::test_result_failed,
279 "Exited with code 1");
281 store::results_iterator iter = transaction.get_results();
283 ATF_REQUIRE_EQ(test_program_6, *iter.test_program());
284 ATF_REQUIRE_EQ("this_fails", iter.test_case_name());
285 ATF_REQUIRE_EQ(result_7, iter.result());
286 ATF_REQUIRE(iter.stdout_contents().empty());
287 ATF_REQUIRE(iter.stderr_contents().empty());
288 ATF_REQUIRE_EQ(1357648719000000LL, iter.start_time().to_microseconds());
289 ATF_REQUIRE_EQ(1357648720897182LL, iter.end_time().to_microseconds());
293 ATF_REQUIRE_EQ(test_program_6, *iter.test_program());
294 ATF_REQUIRE_EQ("this_passes", iter.test_case_name());
295 ATF_REQUIRE_EQ(result_6, iter.result());
296 ATF_REQUIRE(iter.stdout_contents().empty());
297 ATF_REQUIRE(iter.stderr_contents().empty());
298 ATF_REQUIRE_EQ(1357648712000000LL, iter.start_time().to_microseconds());
299 ATF_REQUIRE_EQ(1357648718000000LL, iter.end_time().to_microseconds());
303 ATF_REQUIRE_EQ(test_program_6, *iter.test_program());
304 ATF_REQUIRE_EQ("this_skips", iter.test_case_name());
305 ATF_REQUIRE_EQ(result_8, iter.result());
306 ATF_REQUIRE_EQ("Another stdout", iter.stdout_contents());
307 ATF_REQUIRE(iter.stderr_contents().empty());
308 ATF_REQUIRE_EQ(1357648729182013LL, iter.start_time().to_microseconds());
309 ATF_REQUIRE_EQ(1357648730000000LL, iter.end_time().to_microseconds());
313 ATF_REQUIRE_EQ(test_program_7, *iter.test_program());
314 ATF_REQUIRE_EQ("main", iter.test_case_name());
315 ATF_REQUIRE_EQ(result_9, iter.result());
316 ATF_REQUIRE(iter.stdout_contents().empty());
317 ATF_REQUIRE_EQ("Another stderr", iter.stderr_contents());
318 ATF_REQUIRE_EQ(1357648740120000LL, iter.start_time().to_microseconds());
319 ATF_REQUIRE_EQ(1357648750081700LL, iter.end_time().to_microseconds());
326 /// Validates the contents of the action with identifier 4.
328 /// \param dbpath Path to the database in which to check the action contents.
330 check_action_4(const fs::path& dbpath)
332 store::read_backend backend = store::read_backend::open_ro(dbpath);
333 store::read_transaction transaction = backend.start_read();
335 const fs::path root("/usr/tests");
336 std::map< std::string, std::string > environment;
337 environment["LANG"] = "C";
338 environment["PATH"] = "/bin:/usr/bin";
339 environment["TERM"] = "xterm";
340 const model::context context(root, environment);
342 ATF_REQUIRE_EQ(context, transaction.get_context());
344 const model::test_program test_program_8 = model::test_program_builder(
345 "plain", fs::path("subdir/another_test"), fs::path("/usr/tests"),
347 .add_test_case("main",
348 model::metadata_builder()
349 .set_timeout(datetime::delta(10, 0))
351 .set_metadata(model::metadata_builder()
352 .set_timeout(datetime::delta(10, 0))
355 const model::test_result result_10(model::test_result_failed,
358 const model::test_program test_program_9 = model::test_program_builder(
359 "atf", fs::path("complex_test"), fs::path("/usr/tests"),
361 .add_test_case("this_passes")
362 .add_test_case("this_fails",
363 model::metadata_builder()
364 .set_description("Test description")
365 .set_required_user("root")
368 const model::test_result result_11(model::test_result_passed);
369 const model::test_result result_12(model::test_result_failed,
372 store::results_iterator iter = transaction.get_results();
374 ATF_REQUIRE_EQ(test_program_9, *iter.test_program());
375 ATF_REQUIRE_EQ("this_fails", iter.test_case_name());
376 ATF_REQUIRE_EQ(result_12, iter.result());
377 ATF_REQUIRE(iter.stdout_contents().empty());
378 ATF_REQUIRE(iter.stderr_contents().empty());
379 ATF_REQUIRE_EQ(1357644397100000LL, iter.start_time().to_microseconds());
380 ATF_REQUIRE_EQ(1357644399005000LL, iter.end_time().to_microseconds());
384 ATF_REQUIRE_EQ(test_program_9, *iter.test_program());
385 ATF_REQUIRE_EQ("this_passes", iter.test_case_name());
386 ATF_REQUIRE_EQ(result_11, iter.result());
387 ATF_REQUIRE(iter.stdout_contents().empty());
388 ATF_REQUIRE(iter.stderr_contents().empty());
389 ATF_REQUIRE_EQ(1357644396500000LL, iter.start_time().to_microseconds());
390 ATF_REQUIRE_EQ(1357644397000000LL, iter.end_time().to_microseconds());
394 ATF_REQUIRE_EQ(test_program_8, *iter.test_program());
395 ATF_REQUIRE_EQ("main", iter.test_case_name());
396 ATF_REQUIRE_EQ(result_10, iter.result());
397 ATF_REQUIRE_EQ("Test stdout", iter.stdout_contents());
398 ATF_REQUIRE_EQ("Test stderr", iter.stderr_contents());
399 ATF_REQUIRE_EQ(1357644395000000LL, iter.start_time().to_microseconds());
400 ATF_REQUIRE_EQ(1357644396000000LL, iter.end_time().to_microseconds());
407 } // anonymous namespace
410 #define CURRENT_SCHEMA_TEST(dataset) \
411 ATF_TEST_CASE(current_schema_ ##dataset); \
412 ATF_TEST_CASE_HEAD(current_schema_ ##dataset) \
414 logging::set_inmemory(); \
415 const std::string required_files = \
416 store::detail::schema_file().str() + " " + \
417 testdata_file("testdata_v3_" #dataset ".sql").str(); \
418 set_md_var("require.files", required_files); \
420 ATF_TEST_CASE_BODY(current_schema_ ##dataset) \
422 const fs::path testpath("test.db"); \
424 sqlite::database db = sqlite::database::open( \
425 testpath, sqlite::open_readwrite | sqlite::open_create); \
426 db.exec(utils::read_file(store::detail::schema_file())); \
427 db.exec(utils::read_file(testdata_file(\
428 "testdata_v3_" #dataset ".sql"))); \
431 check_action_ ## dataset (testpath); \
433 CURRENT_SCHEMA_TEST(1);
434 CURRENT_SCHEMA_TEST(2);
435 CURRENT_SCHEMA_TEST(3);
436 CURRENT_SCHEMA_TEST(4);
439 #define MIGRATE_SCHEMA_TEST(from_version) \
440 ATF_TEST_CASE(migrate_schema__from_v ##from_version); \
441 ATF_TEST_CASE_HEAD(migrate_schema__from_v ##from_version) \
443 logging::set_inmemory(); \
445 const char* schema = "schema_v" #from_version ".sql"; \
446 const char* testdata = "testdata_v" #from_version ".sql"; \
448 std::string required_files = \
449 testdata_file(schema).str() + " " + testdata_file(testdata).str(); \
450 for (int i = from_version; i < store::detail::current_schema_version; \
452 required_files += " " + store::detail::migration_file( \
455 set_md_var("require.files", required_files); \
457 ATF_TEST_CASE_BODY(migrate_schema__from_v ##from_version) \
459 const char* schema = "schema_v" #from_version ".sql"; \
460 const char* testdata = "testdata_v" #from_version ".sql"; \
462 const fs::path testpath("test.db"); \
464 sqlite::database db = sqlite::database::open( \
465 testpath, sqlite::open_readwrite | sqlite::open_create); \
466 db.exec(utils::read_file(testdata_file(schema))); \
467 db.exec(utils::read_file(testdata_file(testdata))); \
470 store::migrate_schema(fs::path("test.db")); \
472 check_action_2(fs::path(".kyua/store/" \
473 "results.test_suite_root.20130108-111331-000000.db")); \
474 check_action_3(fs::path(".kyua/store/" \
475 "results.usr_tests.20130108-123832-000000.db")); \
476 check_action_4(fs::path(".kyua/store/" \
477 "results.usr_tests.20130108-112635-000000.db")); \
479 MIGRATE_SCHEMA_TEST(1);
480 MIGRATE_SCHEMA_TEST(2);
483 ATF_INIT_TEST_CASES(tcs)
485 ATF_ADD_TEST_CASE(tcs, current_schema_1);
486 ATF_ADD_TEST_CASE(tcs, current_schema_2);
487 ATF_ADD_TEST_CASE(tcs, current_schema_3);
488 ATF_ADD_TEST_CASE(tcs, current_schema_4);
490 ATF_ADD_TEST_CASE(tcs, migrate_schema__from_v1);
491 ATF_ADD_TEST_CASE(tcs, migrate_schema__from_v2);