1 // Copyright (c) 2008 The NetBSD Foundation, Inc.
2 // All rights reserved.
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions
7 // 1. Redistributions of source code must retain the above copyright
8 // notice, this list of conditions and the following disclaimer.
9 // 2. Redistributions in binary form must reproduce the above copyright
10 // notice, this list of conditions and the following disclaimer in the
11 // documentation and/or other materials provided with the distribution.
13 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
14 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
15 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
16 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
18 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
20 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
24 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 #include <sys/types.h>
46 #include "atf-c++/check.hpp"
47 #include "atf-c++/detail/application.hpp"
48 #include "atf-c++/detail/auto_array.hpp"
49 #include "atf-c++/detail/env.hpp"
50 #include "atf-c++/detail/exceptions.hpp"
51 #include "atf-c++/detail/fs.hpp"
52 #include "atf-c++/detail/process.hpp"
53 #include "atf-c++/detail/sanity.hpp"
54 #include "atf-c++/detail/text.hpp"
56 // ------------------------------------------------------------------------
57 // Auxiliary functions.
58 // ------------------------------------------------------------------------
73 status_check(const status_check_t& p_type, const bool p_negated,
96 output_check(const output_check_t& p_type, const bool p_negated,
97 const std::string& p_value) :
105 class temp_file : public std::ostream {
106 std::auto_ptr< atf::fs::path > m_path;
110 temp_file(const char* pattern) :
114 const atf::fs::path file = atf::fs::path(
115 atf::env::get("TMPDIR", "/tmp")) / pattern;
117 atf::auto_array< char > buf(new char[file.str().length() + 1]);
118 std::strcpy(buf.get(), file.c_str());
120 m_fd = ::mkstemp(buf.get());
122 throw atf::system_error("atf_check::temp_file::temp_file(" +
123 file.str() + ")", "mkstemp(3) failed",
126 m_path.reset(new atf::fs::path(buf.get()));
134 } catch (const atf::system_error&) {
135 // Ignore deletion errors.
146 write(const std::string& text)
148 if (::write(m_fd, text.c_str(), text.size()) == -1)
149 throw atf::system_error("atf_check", "write(2) failed", errno);
163 } // anonymous namespace
166 parse_exit_code(const std::string& str)
169 const int value = atf::text::to_type< int >(str);
170 if (value < 0 || value > 255)
171 throw std::runtime_error("Unused reason");
173 } catch (const std::runtime_error&) {
174 throw atf::application::usage_error("Invalid exit code for -s option; "
175 "must be an integer in range 0-255");
179 static struct name_number {
182 } signal_names_to_numbers[] = {
199 signal_name_to_number(const std::string& str)
201 struct name_number* iter = signal_names_to_numbers;
203 while (signo == INT_MIN && iter->name != NULL) {
204 if (str == iter->name || str == std::string("sig") + iter->name)
213 parse_signal(const std::string& str)
215 const int signo = signal_name_to_number(str);
216 if (signo == INT_MIN) {
218 return atf::text::to_type< int >(str);
219 } catch (std::runtime_error) {
220 throw atf::application::usage_error("Invalid signal name or number "
224 INV(signo != INT_MIN);
229 parse_status_check_arg(const std::string& arg)
231 const std::string::size_type delimiter = arg.find(':');
232 bool negated = (arg.compare(0, 4, "not-") == 0);
233 const std::string action_str = arg.substr(0, delimiter);
234 const std::string action = negated ? action_str.substr(4) : action_str;
235 const std::string value_str = (
236 delimiter == std::string::npos ? "" : arg.substr(delimiter + 1));
240 if (action == "eq") {
241 // Deprecated; use exit instead. TODO: Remove after 0.10.
244 throw atf::application::usage_error("Cannot negate eq checker");
246 value = parse_exit_code(value_str);
247 } else if (action == "exit") {
249 if (value_str.empty())
252 value = parse_exit_code(value_str);
253 } else if (action == "ignore") {
255 throw atf::application::usage_error("Cannot negate ignore checker");
258 } else if (action == "ne") {
259 // Deprecated; use not-exit instead. TODO: Remove after 0.10.
262 throw atf::application::usage_error("Cannot negate ne checker");
264 value = parse_exit_code(value_str);
265 } else if (action == "signal") {
267 if (value_str.empty())
270 value = parse_signal(value_str);
272 throw atf::application::usage_error("Invalid status checker");
274 return status_check(type, negated, value);
279 parse_output_check_arg(const std::string& arg)
281 const std::string::size_type delimiter = arg.find(':');
282 const bool negated = (arg.compare(0, 4, "not-") == 0);
283 const std::string action_str = arg.substr(0, delimiter);
284 const std::string action = negated ? action_str.substr(4) : action_str;
287 if (action == "empty")
289 else if (action == "file")
291 else if (action == "ignore") {
293 throw atf::application::usage_error("Cannot negate ignore checker");
295 } else if (action == "inline")
297 else if (action == "match")
299 else if (action == "save") {
301 throw atf::application::usage_error("Cannot negate save checker");
304 throw atf::application::usage_error("Invalid output checker");
306 return output_check(type, negated, arg.substr(delimiter + 1));
311 flatten_argv(char* const* argv)
315 char* const* arg = &argv[0];
316 while (*arg != NULL) {
329 std::auto_ptr< atf::check::check_result >
330 execute(const char* const* argv)
332 // TODO: This should go to stderr... but fixing it now may be hard as test
333 // cases out there might be relying on stderr being silent.
334 std::cout << "Executing command [ ";
335 for (int i = 0; argv[i] != NULL; ++i)
336 std::cout << argv[i] << " ";
340 atf::process::argv_array argva(argv);
341 return atf::check::exec(argva);
345 std::auto_ptr< atf::check::check_result >
346 execute_with_shell(char* const* argv)
348 const std::string cmd = flatten_argv(argv);
349 const std::string shell = atf::env::get("ATF_SHELL", ATF_SHELL);
351 const char* sh_argv[4];
352 sh_argv[0] = shell.c_str();
354 sh_argv[2] = cmd.c_str();
356 return execute(sh_argv);
361 cat_file(const atf::fs::path& path)
363 std::ifstream stream(path.c_str());
365 throw std::runtime_error("Failed to open " + path.str());
367 stream >> std::noskipws;
368 std::istream_iterator< char > begin(stream), end;
369 std::ostream_iterator< char > out(std::cerr);
370 std::copy(begin, end, out);
377 grep_file(const atf::fs::path& path, const std::string& regexp)
379 std::ifstream stream(path.c_str());
381 throw std::runtime_error("Failed to open " + path.str());
386 while (!found && !std::getline(stream, line).fail()) {
387 if (atf::text::match(line, regexp))
398 file_empty(const atf::fs::path& p)
400 atf::fs::file_info f(p);
402 return (f.get_size() == 0);
406 compare_files(const atf::fs::path& p1, const atf::fs::path& p2)
410 std::ifstream f1(p1.c_str());
412 throw std::runtime_error("Failed to open " + p1.str());
414 std::ifstream f2(p2.c_str());
416 throw std::runtime_error("Failed to open " + p1.str());
419 char buf1[512], buf2[512];
421 f1.read(buf1, sizeof(buf1));
423 throw std::runtime_error("Failed to read from " + p1.str());
425 f2.read(buf2, sizeof(buf2));
427 throw std::runtime_error("Failed to read from " + p1.str());
429 if ((f1.gcount() == 0) && (f2.gcount() == 0)) {
434 if ((f1.gcount() != f2.gcount()) ||
435 (std::memcmp(buf1, buf2, f1.gcount()) != 0)) {
445 print_diff(const atf::fs::path& p1, const atf::fs::path& p2)
447 const atf::process::status s =
448 atf::process::exec(atf::fs::path("diff"),
449 atf::process::argv_array("diff", "-u", p1.c_str(),
451 atf::process::stream_connect(STDOUT_FILENO,
453 atf::process::stream_inherit());
456 std::cerr << "Failed to run diff(3)\n";
458 if (s.exitstatus() != 1)
459 std::cerr << "Error while running diff(3)\n";
464 decode(const std::string& s)
469 res.reserve(s.length());
472 while (i < s.length()) {
477 case 'a': c = '\a'; break;
478 case 'b': c = '\b'; break;
480 case 'e': c = 033; break;
481 case 'f': c = '\f'; break;
482 case 'n': c = '\n'; break;
483 case 'r': c = '\r'; break;
484 case 't': c = '\t'; break;
485 case 'v': c = '\v'; break;
491 while (--count >= 0 && (unsigned)(s[i] - '0') < 8)
492 c = (c << 3) + (s[i++] - '0');
509 run_status_check(const status_check& sc, const atf::check::check_result& cr)
513 if (sc.type == sc_exit) {
514 if (cr.exited() && sc.value != INT_MIN) {
515 const int status = cr.exitcode();
517 if (!sc.negated && sc.value != status) {
518 std::cerr << "Fail: incorrect exit status: "
519 << status << ", expected: "
522 } else if (sc.negated && sc.value == status) {
523 std::cerr << "Fail: incorrect exit status: "
524 << status << ", expected: "
525 << "anything else\n";
529 } else if (cr.exited() && sc.value == INT_MIN) {
532 std::cerr << "Fail: program did not exit cleanly\n";
535 } else if (sc.type == sc_ignore) {
537 } else if (sc.type == sc_signal) {
538 if (cr.signaled() && sc.value != INT_MIN) {
539 const int status = cr.termsig();
541 if (!sc.negated && sc.value != status) {
542 std::cerr << "Fail: incorrect signal received: "
543 << status << ", expected: " << sc.value << "\n";
545 } else if (sc.negated && sc.value == status) {
546 std::cerr << "Fail: incorrect signal received: "
547 << status << ", expected: "
548 << "anything else\n";
552 } else if (cr.signaled() && sc.value == INT_MIN) {
555 std::cerr << "Fail: program did not receive a signal\n";
563 if (result == false) {
564 std::cerr << "stdout:\n";
565 cat_file(atf::fs::path(cr.stdout_path()));
568 std::cerr << "stderr:\n";
569 cat_file(atf::fs::path(cr.stderr_path()));
578 run_status_checks(const std::vector< status_check >& checks,
579 const atf::check::check_result& result)
583 for (std::vector< status_check >::const_iterator iter = checks.begin();
584 !ok && iter != checks.end(); iter++) {
585 ok |= run_status_check(*iter, result);
593 run_output_check(const output_check oc, const atf::fs::path& path,
594 const std::string& stdxxx)
598 if (oc.type == oc_empty) {
599 const bool is_empty = file_empty(path);
600 if (!oc.negated && !is_empty) {
601 std::cerr << "Fail: " << stdxxx << " not empty\n";
602 print_diff(atf::fs::path("/dev/null"), path);
604 } else if (oc.negated && is_empty) {
605 std::cerr << "Fail: " << stdxxx << " is empty\n";
609 } else if (oc.type == oc_file) {
610 const bool equals = compare_files(path, atf::fs::path(oc.value));
611 if (!oc.negated && !equals) {
612 std::cerr << "Fail: " << stdxxx << " does not match golden "
614 print_diff(atf::fs::path(oc.value), path);
616 } else if (oc.negated && equals) {
617 std::cerr << "Fail: " << stdxxx << " matches golden output\n";
618 cat_file(atf::fs::path(oc.value));
622 } else if (oc.type == oc_ignore) {
624 } else if (oc.type == oc_inline) {
625 temp_file temp("atf-check.XXXXXX");
626 temp.write(decode(oc.value));
629 const bool equals = compare_files(path, temp.get_path());
630 if (!oc.negated && !equals) {
631 std::cerr << "Fail: " << stdxxx << " does not match expected "
633 print_diff(temp.get_path(), path);
635 } else if (oc.negated && equals) {
636 std::cerr << "Fail: " << stdxxx << " matches expected value\n";
637 cat_file(temp.get_path());
641 } else if (oc.type == oc_match) {
642 const bool matches = grep_file(path, oc.value);
643 if (!oc.negated && !matches) {
644 std::cerr << "Fail: regexp " + oc.value + " not in " << stdxxx
648 } else if (oc.negated && matches) {
649 std::cerr << "Fail: regexp " + oc.value + " is in " << stdxxx
655 } else if (oc.type == oc_save) {
657 std::ifstream ifs(path.c_str(), std::fstream::binary);
658 ifs >> std::noskipws;
659 std::istream_iterator< char > begin(ifs), end;
661 std::ofstream ofs(oc.value.c_str(), std::fstream::binary
662 | std::fstream::trunc);
663 std::ostream_iterator <char> obegin(ofs);
665 std::copy(begin, end, obegin);
677 run_output_checks(const std::vector< output_check >& checks,
678 const atf::fs::path& path, const std::string& stdxxx)
682 for (std::vector< output_check >::const_iterator iter = checks.begin();
683 iter != checks.end(); iter++) {
684 ok &= run_output_check(*iter, path, stdxxx);
690 // ------------------------------------------------------------------------
691 // The "atf_check" application.
692 // ------------------------------------------------------------------------
696 class atf_check : public atf::application::app {
699 std::vector< status_check > m_status_checks;
700 std::vector< output_check > m_stdout_checks;
701 std::vector< output_check > m_stderr_checks;
703 static const char* m_description;
705 bool run_output_checks(const atf::check::check_result&,
706 const std::string&) const;
708 std::string specific_args(void) const;
709 options_set specific_options(void) const;
710 void process_option(int, const char*);
711 void process_option_s(const std::string&);
718 } // anonymous namespace
720 const char* atf_check::m_description =
721 "atf-check executes given command and analyzes its results.";
723 atf_check::atf_check(void) :
724 app(m_description, "atf-check(1)"),
730 atf_check::run_output_checks(const atf::check::check_result& r,
731 const std::string& stdxxx)
734 if (stdxxx == "stdout") {
735 return ::run_output_checks(m_stdout_checks,
736 atf::fs::path(r.stdout_path()), "stdout");
737 } else if (stdxxx == "stderr") {
738 return ::run_output_checks(m_stderr_checks,
739 atf::fs::path(r.stderr_path()), "stderr");
747 atf_check::specific_args(void)
753 atf_check::options_set
754 atf_check::specific_options(void)
757 using atf::application::option;
760 opts.insert(option('s', "qual:value", "Handle status. Qualifier "
761 "must be one of: ignore exit:<num> signal:<name|num>"));
762 opts.insert(option('o', "action:arg", "Handle stdout. Action must be "
763 "one of: empty ignore file:<path> inline:<val> match:regexp "
765 opts.insert(option('e', "action:arg", "Handle stderr. Action must be "
766 "one of: empty ignore file:<path> inline:<val> match:regexp "
768 opts.insert(option('x', "", "Execute command as a shell command"));
774 atf_check::process_option(int ch, const char* arg)
778 m_status_checks.push_back(parse_status_check_arg(arg));
782 m_stdout_checks.push_back(parse_output_check_arg(arg));
786 m_stderr_checks.push_back(parse_output_check_arg(arg));
799 atf_check::main(void)
802 throw atf::application::usage_error("No command specified");
804 int status = EXIT_FAILURE;
806 std::auto_ptr< atf::check::check_result > r =
807 m_xflag ? execute_with_shell(m_argv) : execute(m_argv);
809 if (m_status_checks.empty())
810 m_status_checks.push_back(status_check(sc_exit, false, EXIT_SUCCESS));
811 else if (m_status_checks.size() > 1) {
812 // TODO: Remove this restriction.
813 throw atf::application::usage_error("Cannot specify -s more than once");
816 if (m_stdout_checks.empty())
817 m_stdout_checks.push_back(output_check(oc_empty, false, ""));
818 if (m_stderr_checks.empty())
819 m_stderr_checks.push_back(output_check(oc_empty, false, ""));
821 if ((run_status_checks(m_status_checks, *r) == false) ||
822 (run_output_checks(*r, "stderr") == false) ||
823 (run_output_checks(*r, "stdout") == false))
824 status = EXIT_FAILURE;
826 status = EXIT_SUCCESS;
832 main(int argc, char* const* argv)
834 return atf_check().run(argc, argv);