1 # Copyright 2012 The Kyua Authors.
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above copyright
11 # notice, this list of conditions and the following disclaimer in the
12 # documentation and/or other materials provided with the distribution.
13 # * Neither the name of Google Inc. nor the names of its contributors
14 # may be used to endorse or promote products derived from this software
15 # without specific prior written permission.
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 # Executes a mock test suite to generate data in the database.
32 # \param mock_env The value to store in a MOCK variable in the environment.
33 # Use this to be able to differentiate executions by inspecting the
34 # context of the output.
35 # \param dbfile_name File to which to write the path to the generated database
38 local mock_env="${1}"; shift
39 local dbfile_name="${1}"; shift
43 test_suite("integration")
44 atf_test_program{name="simple_all_pass"}
45 atf_test_program{name="simple_some_fail"}
46 atf_test_program{name="metadata"}
49 utils_cp_helper simple_all_pass .
50 utils_cp_helper simple_some_fail .
51 utils_cp_helper metadata .
52 atf_check -s exit:1 -o save:stdout -e empty env MOCK="${mock_env}" kyua test
53 grep '^Results saved to ' stdout | cut -d ' ' -f 4 >"${dbfile_name}"
56 # Ensure the results of 'report-html' come from the database.
57 rm Kyuafile simple_all_pass simple_some_fail metadata
61 # Ensure a file has a set of strings.
63 # \param file The name of the file to check.
64 # \param ... List of strings to check.
66 local file="${1}"; shift
68 while [ ${#} -gt 0 ]; do
69 echo "Checking for presence of '${1}' in ${file}"
70 if grep "${1}" "${file}" >/dev/null; then
73 atf_fail "Test case output not found in HTML page ${file}"
80 # Ensure a file does not have a set of strings.
82 # \param file The name of the file to check.
83 # \param ... List of strings to check.
85 local file="${1}"; shift
87 while [ ${#} -gt 0 ]; do
88 echo "Checking for lack of '${1}' in ${file}"
89 if grep "${1}" "${file}" >/dev/null; then
90 atf_fail "Spurious test case output found in HTML page"
97 utils_test_case default_behavior__ok
98 default_behavior__ok_body() {
99 run_tests "mock1" unused_dbfile_name
101 atf_check -s exit:0 -o ignore -e empty kyua report-html
105 html/simple_all_pass_skip.html \
106 html/simple_some_fail_fail.html
108 test -f "${f}" || atf_fail "Missing ${f}"
111 atf_check -o match:"2 TESTS FAILING" cat html/index.html
113 check_in_file html/simple_all_pass_skip.html \
114 "This is the stdout of skip" "This is the stderr of skip"
115 check_not_in_file html/simple_all_pass_skip.html \
116 "This is the stdout of pass" "This is the stderr of pass" \
117 "This is the stdout of fail" "This is the stderr of fail" \
118 "Test case did not write anything to"
120 check_in_file html/simple_some_fail_fail.html \
121 "This is the stdout of fail" "This is the stderr of fail"
122 check_not_in_file html/simple_some_fail_fail.html \
123 "This is the stdout of pass" "This is the stderr of pass" \
124 "This is the stdout of skip" "This is the stderr of skip" \
125 "Test case did not write anything to"
127 check_in_file html/metadata_one_property.html \
128 "description = Does nothing but has one metadata property"
129 check_not_in_file html/metadata_one_property.html \
130 "allowed_architectures = some-architecture"
132 check_in_file html/metadata_many_properties.html \
133 "allowed_architectures = some-architecture"
134 check_not_in_file html/metadata_many_properties.html \
135 "description = Does nothing but has one metadata property"
139 utils_test_case default_behavior__no_store
140 default_behavior__no_store_body() {
141 echo 'kyua: E: No previous results file found for test suite' \
142 "$(utils_test_suite_id)." >experr
143 atf_check -s exit:2 -o empty -e file:experr kyua report-html
147 utils_test_case results_file__explicit
148 results_file__explicit_body() {
149 run_tests "mock1" dbfile_name1
150 run_tests "mock2" dbfile_name2
152 atf_check -s exit:0 -o ignore -e empty kyua report-html \
153 --results-file="$(cat dbfile_name1)"
154 grep "MOCK.*mock1" html/context.html || atf_fail "Invalid context in report"
157 atf_check -s exit:0 -o ignore -e empty kyua report-html \
158 --results-file="$(cat dbfile_name2)"
159 grep "MOCK.*mock2" html/context.html || atf_fail "Invalid context in report"
163 utils_test_case results_file__not_found
164 results_file__not_found_body() {
165 atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \
166 kyua report-html --results-file=foo
170 utils_test_case force__yes
172 run_tests "mock1" unused_dbfile_name
174 atf_check -s exit:0 -o ignore -e empty kyua report-html
175 test -f html/index.html || atf_fail "Expected file not created"
177 atf_check -s exit:0 -o ignore -e empty kyua report-html --force
178 test -f html/index.html || atf_fail "Expected file not created"
182 utils_test_case force__no
184 run_tests "mock1" unused_dbfile_name
186 atf_check -s exit:0 -o ignore -e empty kyua report-html
187 test -f html/index.html || atf_fail "Expected file not created"
191 kyua: E: Output directory 'html' already exists; maybe use --force?.
193 atf_check -s exit:2 -o empty -e file:experr kyua report-html
194 test ! -f html/index.html || atf_fail "Not expected file created"
198 utils_test_case output__explicit
199 output__explicit_body() {
200 run_tests "mock1" unused_dbfile_name
203 atf_check -s exit:0 -o ignore -e empty kyua report-html --output=output/foo
204 test ! -d html || atf_fail "Not expected directory created"
205 test -f output/foo/index.html || atf_fail "Expected file not created"
209 utils_test_case results_filter__ok
210 results_filter__ok_body() {
211 run_tests "mock1" unused_dbfile_name
213 atf_check -s exit:0 -o ignore -e empty kyua report-html \
214 --results-filter=passed
218 html/simple_all_pass_pass.html \
219 html/simple_some_fail_pass.html \
220 html/metadata_no_properties.html \
221 html/metadata_with_cleanup.html
223 test -f "${f}" || atf_fail "Missing ${f}"
226 atf_check -o match:"2 TESTS FAILING" cat html/index.html
228 check_in_file html/simple_all_pass_pass.html \
229 "This is the stdout of pass" "This is the stderr of pass"
230 check_not_in_file html/simple_all_pass_pass.html \
231 "This is the stdout of skip" "This is the stderr of skip" \
232 "This is the stdout of fail" "This is the stderr of fail" \
233 "Test case did not write anything to"
235 check_in_file html/simple_some_fail_pass.html \
236 "Test case did not write anything to stdout" \
237 "Test case did not write anything to stderr"
238 check_not_in_file html/simple_some_fail_pass.html \
239 "This is the stdout of pass" "This is the stderr of pass" \
240 "This is the stdout of skip" "This is the stderr of skip" \
241 "This is the stdout of fail" "This is the stderr of fail"
245 utils_test_case results_filter__invalid
246 results_filter__invalid_body() {
247 echo "kyua: E: Unknown result type 'foo-bar'." >experr
248 atf_check -s exit:2 -o empty -e file:experr kyua report-html \
249 --results-filter=passed,foo-bar
253 atf_init_test_cases() {
254 atf_add_test_case default_behavior__ok
255 atf_add_test_case default_behavior__no_store
257 atf_add_test_case results_file__explicit
258 atf_add_test_case results_file__not_found
260 atf_add_test_case force__yes
261 atf_add_test_case force__no
263 atf_add_test_case output__explicit
265 atf_add_test_case results_filter__ok
266 atf_add_test_case results_filter__invalid