1 # Copyright 2011 The Kyua Authors.
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above copyright
11 # notice, this list of conditions and the following disclaimer in the
12 # documentation and/or other materials provided with the distribution.
13 # * Neither the name of Google Inc. nor the names of its contributors
14 # may be used to endorse or promote products derived from this software
15 # without specific prior written permission.
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 utils_test_case no_args
34 test_suite("integration")
35 atf_test_program{name="metadata"}
36 atf_test_program{name="simple_all_pass"}
37 include("subdir/Kyuafile")
39 utils_cp_helper metadata .
40 utils_cp_helper simple_all_pass .
43 cat >subdir/Kyuafile <<EOF
45 test_suite("integration2")
46 atf_test_program{name="simple_some_fail"}
48 utils_cp_helper simple_some_fail subdir
51 metadata:many_properties
52 metadata:no_properties
57 subdir/simple_some_fail:fail
58 subdir/simple_some_fail:pass
60 atf_check -s exit:0 -o file:expout -e empty kyua list
64 utils_test_case one_arg__subdir
65 one_arg__subdir_body() {
68 test_suite("top-level")
69 include("subdir/Kyuafile")
73 cat >subdir/Kyuafile <<EOF
75 test_suite("in-subdir")
76 atf_test_program{name="simple_all_pass"}
78 utils_cp_helper simple_all_pass subdir
81 subdir/simple_all_pass:pass
82 subdir/simple_all_pass:skip
84 atf_check -s exit:0 -o file:expout -e empty kyua list subdir
88 utils_test_case one_arg__test_case
89 one_arg__test_case_body() {
92 test_suite("top-level")
93 atf_test_program{name="first"}
94 atf_test_program{name="second"}
96 utils_cp_helper simple_all_pass first
97 utils_cp_helper simple_all_pass second
102 atf_check -s exit:0 -o file:expout -e empty kyua list first:skip
106 utils_test_case one_arg__test_program
107 one_arg__test_program_body() {
110 test_suite("top-level")
111 atf_test_program{name="first"}
112 atf_test_program{name="second"}
114 utils_cp_helper simple_all_pass first
115 utils_cp_helper simple_some_fail second
121 atf_check -s exit:0 -o file:expout -e empty kyua list second
125 utils_test_case one_arg__invalid
126 one_arg__invalid_body() {
128 kyua: E: Test case component in 'foo:' is empty.
130 atf_check -s exit:2 -o empty -e file:experr kyua list foo:
133 kyua: E: Program name '/a/b' must be relative to the test suite, not absolute.
135 atf_check -s exit:2 -o empty -e file:experr kyua list /a/b
139 utils_test_case many_args__ok
140 many_args__ok_body() {
143 test_suite("top-level")
144 include("subdir/Kyuafile")
145 atf_test_program{name="first"}
147 utils_cp_helper simple_all_pass first
150 cat >subdir/Kyuafile <<EOF
152 test_suite("in-subdir")
153 atf_test_program{name="second"}
155 utils_cp_helper simple_some_fail subdir/second
158 subdir/second:fail (in-subdir)
159 subdir/second:pass (in-subdir)
160 first:pass (top-level)
162 atf_check -s exit:0 -o file:expout -e empty kyua list -v subdir first:pass
166 utils_test_case many_args__invalid
167 many_args__invalid_body() {
169 kyua: E: Program name component in ':badbad' is empty.
171 atf_check -s exit:2 -o empty -e file:experr kyua list this-is-ok :badbad
174 kyua: E: Program name '/foo' must be relative to the test suite, not absolute.
176 atf_check -s exit:2 -o empty -e file:experr kyua list this-is-ok /foo
180 utils_test_case many_args__no_match__all
181 many_args__no_match__all_body() {
184 test_suite("top-level")
185 atf_test_program{name="first"}
186 atf_test_program{name="second"}
188 utils_cp_helper simple_all_pass first
189 utils_cp_helper simple_all_pass second
192 kyua: W: No test cases matched by the filter 'first1'.
194 atf_check -s exit:1 -o empty -e file:experr kyua list first1
198 utils_test_case many_args__no_match__some
199 many_args__no_match__some_body() {
202 test_suite("top-level")
203 atf_test_program{name="first"}
204 atf_test_program{name="second"}
205 atf_test_program{name="third"}
207 utils_cp_helper simple_all_pass first
208 utils_cp_helper simple_all_pass second
209 utils_cp_helper simple_some_fail third
219 kyua: W: No test cases matched by the filter 'fifth'.
220 kyua: W: No test cases matched by the filter 'fourth'.
222 atf_check -s exit:1 -o file:expout -e file:experr kyua list first fourth \
227 utils_test_case args_are_relative
228 args_are_relative_body() {
230 cat >root/Kyuafile <<EOF
232 test_suite("integration-1")
233 atf_test_program{name="first"}
234 atf_test_program{name="second"}
235 include("subdir/Kyuafile")
237 utils_cp_helper simple_all_pass root/first
238 utils_cp_helper simple_some_fail root/second
241 cat >root/subdir/Kyuafile <<EOF
243 test_suite("integration-2")
244 atf_test_program{name="third"}
245 atf_test_program{name="fourth"}
247 utils_cp_helper simple_all_pass root/subdir/third
248 utils_cp_helper simple_some_fail root/subdir/fourth
251 first:pass (integration-1)
252 first:skip (integration-1)
253 subdir/fourth:fail (integration-2)
255 atf_check -s exit:0 -o file:expout -e empty kyua list \
256 -v -k "$(pwd)/root/Kyuafile" first subdir/fourth:fail
260 utils_test_case only_load_used_test_programs
261 only_load_used_test_programs_body() {
264 test_suite("integration")
265 atf_test_program{name="first"}
266 atf_test_program{name="second"}
268 utils_cp_helper simple_all_pass first
269 utils_cp_helper bad_test_program second
275 CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE
276 atf_check -s exit:0 -o file:expout -e empty kyua list first
277 if [ -f "${CREATE_COOKIE}" ]; then
278 atf_fail "An unmatched test case has been executed, which harms" \
284 utils_test_case config_behavior
285 config_behavior_body() {
286 cat >"my-config" <<EOF
288 test_suites.suite1["the-variable"] = "value1"
293 atf_test_program{name="config1", test_suite="suite1"}
295 utils_cp_helper config config1
297 CONFIG_VAR_FILE="$(pwd)/cookie"; export CONFIG_VAR_FILE
298 if [ -f "${CONFIG_VAR_FILE}" ]; then
299 atf_fail "Cookie file already created; test case list may have gotten" \
300 "a bad configuration"
302 atf_check -s exit:0 -o ignore -e empty kyua -c my-config list
303 [ -f "${CONFIG_VAR_FILE}" ] || \
304 atf_fail "Cookie file not created; test case list did not get" \
305 "configuration variables"
306 value="$(cat "${CONFIG_VAR_FILE}")"
307 [ "${value}" = "value1" ] || \
308 atf_fail "Invalid value (${value}) in cookie file; test case list did" \
309 "not get the correct configuration variables"
313 utils_test_case build_root_flag
314 build_root_flag_body() {
321 test_suite("top-level")
322 include("subdir/Kyuafile")
323 atf_test_program{name="first"}
325 echo 'invalid' >first
326 utils_cp_helper simple_all_pass build/first
328 cat >subdir/Kyuafile <<EOF
330 test_suite("in-subdir")
331 atf_test_program{name="second"}
333 echo 'invalid' >subdir/second
334 utils_cp_helper simple_some_fail build/subdir/second
341 atf_check -s exit:0 -o file:expout -e empty kyua list --build-root=build \
346 utils_test_case kyuafile_flag__no_args
347 kyuafile_flag__no_args_body() {
349 This file is bogus but it is not loaded.
354 test_suite("integration")
355 atf_test_program{name="sometest"}
357 utils_cp_helper simple_all_pass sometest
363 atf_check -s exit:0 -o file:expout -e empty kyua list -k myfile
364 atf_check -s exit:0 -o file:expout -e empty kyua list --kyuafile=myfile
368 utils_test_case kyuafile_flag__some_args
369 kyuafile_flag__some_args_body() {
371 This file is bogus but it is not loaded.
376 test_suite("hello-world")
377 atf_test_program{name="sometest"}
379 utils_cp_helper simple_all_pass sometest
382 sometest:pass (hello-world)
383 sometest:skip (hello-world)
385 atf_check -s exit:0 -o file:expout -e empty kyua list -v -k myfile sometest
386 atf_check -s exit:0 -o file:expout -e empty kyua list -v --kyuafile=myfile \
391 utils_test_case verbose_flag
392 verbose_flag_body() {
395 test_suite("integration-suite-1")
396 atf_test_program{name="simple_all_pass"}
397 plain_test_program{name="i_am_plain", timeout=654}
398 include("subdir/Kyuafile")
400 utils_cp_helper simple_all_pass .
404 cat >subdir/Kyuafile <<EOF
406 test_suite("integration-suite-2")
407 atf_test_program{name="metadata"}
409 utils_cp_helper metadata subdir
412 simple_all_pass:pass (integration-suite-1)
413 simple_all_pass:skip (integration-suite-1)
414 i_am_plain:main (integration-suite-1)
416 subdir/metadata:many_properties (integration-suite-2)
417 allowed_architectures = some-architecture
418 allowed_platforms = some-platform
419 custom.no-meaning = I am a custom variable
420 description = A description with some padding
421 required_configs = var1 var2 var3
422 required_files = /my/file1 /some/other/file
423 required_programs = /nonexistent/bin3 bin1 bin2
425 subdir/metadata:no_properties (integration-suite-2)
426 subdir/metadata:one_property (integration-suite-2)
427 description = Does nothing but has one metadata property
428 subdir/metadata:with_cleanup (integration-suite-2)
432 atf_check -s exit:0 -o file:expout -e empty kyua list -v
433 atf_check -s exit:0 -o file:expout -e empty kyua list --verbose
437 utils_test_case no_test_program_match
438 no_test_program_match_body() {
441 test_suite("integration")
442 atf_test_program{name="first"}
444 utils_cp_helper simple_all_pass first
445 utils_cp_helper simple_all_pass second
448 kyua: W: No test cases matched by the filter 'second'.
450 atf_check -s exit:1 -o empty -e file:experr kyua list second
454 utils_test_case no_test_case_match
455 no_test_case_match_body() {
458 test_suite("integration")
459 atf_test_program{name="first"}
461 utils_cp_helper simple_all_pass first
464 kyua: W: No test cases matched by the filter 'first:foobar'.
466 atf_check -s exit:1 -o empty -e file:experr kyua list first:foobar
470 utils_test_case missing_kyuafile__no_args
471 missing_kyuafile__no_args_body() {
473 kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found.
475 atf_check -s exit:2 -o empty -e file:experr kyua list
479 utils_test_case missing_kyuafile__test_program
480 missing_kyuafile__test_program_body() {
482 cat >subdir/Kyuafile <<EOF
484 test_suite("integration")
485 atf_test_program{name="unused"}
487 utils_cp_helper simple_all_pass subdir/unused
490 kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found.
492 atf_check -s exit:2 -o empty -e file:experr kyua list subdir/unused
496 utils_test_case missing_kyuafile__subdir
497 missing_kyuafile__subdir_body() {
499 cat >subdir/Kyuafile <<EOF
501 test_suite("integration")
502 atf_test_program{name="unused"}
504 utils_cp_helper simple_all_pass subdir/unused
507 kyua: E: Load of 'Kyuafile' failed: File 'Kyuafile' not found.
509 atf_check -s exit:2 -o empty -e file:experr kyua list subdir
513 utils_test_case bogus_kyuafile
514 bogus_kyuafile_body() {
518 atf_check -s exit:2 -o empty \
519 -e match:"Load of 'Kyuafile' failed: .* Kyuafile:2:" kyua list
523 utils_test_case bogus_test_program
524 bogus_test_program_body() {
527 test_suite("integration")
528 atf_test_program{name="crash_on_list"}
529 atf_test_program{name="non_executable"}
531 utils_cp_helper bad_test_program crash_on_list
532 echo 'I am not executable' >non_executable
535 crash_on_list:__test_cases_list__
536 non_executable:__test_cases_list__
538 atf_check -s exit:0 -o file:expout -e empty kyua list
542 utils_test_case missing_test_program
543 missing_test_program_body() {
546 include("subdir/Kyuafile")
549 cat >subdir/Kyuafile <<EOF
551 test_suite("integration")
552 atf_test_program{name="ok"}
553 atf_test_program{name="i-am-missing"}
555 echo 'I should not be touched because the Kyuafile is bogus' >subdir/ok
557 # CHECK_STYLE_DISABLE
559 kyua: E: Load of 'Kyuafile' failed: .*Non-existent test program 'subdir/i-am-missing'.
562 atf_check -s exit:2 -o empty -e "match:$(cat experr)" kyua list
566 atf_init_test_cases() {
567 atf_add_test_case no_args
568 atf_add_test_case one_arg__subdir
569 atf_add_test_case one_arg__test_case
570 atf_add_test_case one_arg__test_program
571 atf_add_test_case one_arg__invalid
572 atf_add_test_case many_args__ok
573 atf_add_test_case many_args__invalid
574 atf_add_test_case many_args__no_match__all
575 atf_add_test_case many_args__no_match__some
577 atf_add_test_case args_are_relative
579 atf_add_test_case only_load_used_test_programs
581 atf_add_test_case config_behavior
583 atf_add_test_case build_root_flag
585 atf_add_test_case kyuafile_flag__no_args
586 atf_add_test_case kyuafile_flag__some_args
588 atf_add_test_case verbose_flag
590 atf_add_test_case no_test_program_match
591 atf_add_test_case no_test_case_match
593 atf_add_test_case missing_kyuafile__no_args
594 atf_add_test_case missing_kyuafile__test_program
595 atf_add_test_case missing_kyuafile__subdir
597 atf_add_test_case bogus_kyuafile
598 atf_add_test_case bogus_test_program
599 atf_add_test_case missing_test_program