2 * Copyright (c) 2003-2007 Tim Kientzle
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 * Various utility routines useful for test programs.
28 * Each test program is linked against this file.
38 * This same file is used pretty much verbatim for all test harnesses.
40 * The next few lines are the only differences.
42 #undef PROGRAM /* Testing a library, not a program. */
43 #define ENVBASE "LIBARCHIVE" /* Prefix for environment variables. */
44 #define EXTRA_DUMP(x) archive_error_string((struct archive *)(x))
45 #define EXTRA_VERSION archive_version()
46 __FBSDID("$FreeBSD$");
49 * "list.h" is simply created by "grep DEFINE_TEST"; it has
51 * DEFINE_TEST(test_function)
53 * Include it here with a suitable DEFINE_TEST to declare all of the
57 #define DEFINE_TEST(name) void name(void);
60 /* Interix doesn't define these in a standard header. */
66 /* Default is to crash and try to force a core dump on failure. */
67 static int dump_on_failure = 1;
68 /* Default is to print some basic information about each test. */
69 static int quiet_flag = 0;
70 /* Cumulative count of component failures. */
71 static int failures = 0;
72 /* Cumulative count of skipped component tests. */
74 /* Cumulative count of assertions. */
75 static int assertions = 0;
77 /* Directory where uuencoded reference files can be found. */
81 * My own implementation of the standard assert() macro emits the
82 * message in the same format as GCC (file:line: message).
83 * It also includes some additional useful information.
84 * This makes it a lot easier to skim through test failures in
87 * It also supports a few special features specifically to simplify
89 * failure(fmt, args) -- Stores a text string that gets
90 * printed if the following assertion fails, good for
91 * explaining subtle tests.
93 static char msg[4096];
96 * For each test source file, we remember how many times each
97 * failure was reported.
99 static const char *failed_filename = NULL;
103 } failed_lines[1000];
106 * Count this failure; return the number of previous failures.
109 previous_failures(const char *filename, int line)
114 if (failed_filename == NULL || strcmp(failed_filename, filename) != 0)
115 memset(failed_lines, 0, sizeof(failed_lines));
116 failed_filename = filename;
118 for (i = 0; i < sizeof(failed_lines)/sizeof(failed_lines[0]); i++) {
119 if (failed_lines[i].line == line) {
120 count = failed_lines[i].count;
121 failed_lines[i].count++;
124 if (failed_lines[i].line == 0) {
125 failed_lines[i].line = line;
126 failed_lines[i].count = 1;
134 * Copy arguments into file-local variables.
136 static const char *test_filename;
137 static int test_line;
138 static void *test_extra;
139 void test_setup(const char *filename, int line)
141 test_filename = filename;
146 * Inform user that we're skipping a test.
149 test_skipping(const char *fmt, ...)
153 if (previous_failures(test_filename, test_line))
157 fprintf(stderr, " *** SKIPPING: ");
158 vfprintf(stderr, fmt, ap);
159 fprintf(stderr, "\n");
164 /* Common handling of failed tests. */
166 report_failure(void *extra)
168 if (msg[0] != '\0') {
169 fprintf(stderr, " Description: %s\n", msg);
175 fprintf(stderr, " detail: %s\n", EXTRA_DUMP(extra));
177 (void)extra; /* UNUSED */
180 if (dump_on_failure) {
182 " *** forcing core dump so failure can be debugged ***\n");
189 * Summarize repeated failures in the just-completed test file.
190 * The reports above suppress multiple failures from the same source
191 * line; this reports on any tests that did fail multiple times.
194 summarize_comparator(const void *a0, const void *b0)
196 const struct line *a = a0, *b = b0;
197 if (a->line == 0 && b->line == 0)
203 return (a->line - b->line);
211 qsort(failed_lines, sizeof(failed_lines)/sizeof(failed_lines[0]),
212 sizeof(failed_lines[0]), summarize_comparator);
213 for (i = 0; i < sizeof(failed_lines)/sizeof(failed_lines[0]); i++) {
214 if (failed_lines[i].line == 0)
216 if (failed_lines[i].count > 1)
217 fprintf(stderr, "%s:%d: Failed %d times\n",
218 failed_filename, failed_lines[i].line,
219 failed_lines[i].count);
221 /* Clear the failure history for the next file. */
222 memset(failed_lines, 0, sizeof(failed_lines));
225 /* Set up a message to display only after a test fails. */
227 failure(const char *fmt, ...)
231 vsprintf(msg, fmt, ap);
235 /* Generic assert() just displays the failed condition. */
237 test_assert(const char *file, int line, int value, const char *condition, void *extra)
245 if (previous_failures(file, line))
247 fprintf(stderr, "%s:%d: Assertion failed\n", file, line);
248 fprintf(stderr, " Condition: %s\n", condition);
249 report_failure(extra);
253 /* assertEqualInt() displays the values of the two integers. */
255 test_assert_equal_int(const char *file, int line,
256 int v1, const char *e1, int v2, const char *e2, void *extra)
264 if (previous_failures(file, line))
266 fprintf(stderr, "%s:%d: Assertion failed: Ints not equal\n",
268 fprintf(stderr, " %s=%d\n", e1, v1);
269 fprintf(stderr, " %s=%d\n", e2, v2);
270 report_failure(extra);
274 /* assertEqualString() displays the values of the two strings. */
276 test_assert_equal_string(const char *file, int line,
277 const char *v1, const char *e1,
278 const char *v2, const char *e2,
282 if (v1 == NULL || v2 == NULL) {
287 } else if (strcmp(v1, v2) == 0) {
292 if (previous_failures(file, line))
294 fprintf(stderr, "%s:%d: Assertion failed: Strings not equal\n",
296 fprintf(stderr, " %s = \"%s\"\n", e1, v1);
297 fprintf(stderr, " %s = \"%s\"\n", e2, v2);
298 report_failure(extra);
302 /* assertEqualWString() displays the values of the two strings. */
304 test_assert_equal_wstring(const char *file, int line,
305 const wchar_t *v1, const char *e1,
306 const wchar_t *v2, const char *e2,
310 if (wcscmp(v1, v2) == 0) {
315 if (previous_failures(file, line))
317 fprintf(stderr, "%s:%d: Assertion failed: Unicode strings not equal\n",
319 fwprintf(stderr, L" %s = \"%ls\"\n", e1, v1);
320 fwprintf(stderr, L" %s = \"%ls\"\n", e2, v2);
321 report_failure(extra);
326 * Pretty standard hexdump routine. As a bonus, if ref != NULL, then
327 * any bytes in p that differ from ref will be highlighted with '_'
328 * before and after the hex value.
331 hexdump(const char *p, const char *ref, size_t l, size_t offset)
336 for(i=0; i < l; i+=16) {
337 fprintf(stderr, "%04x", i + offset);
339 for (j = 0; j < 16 && i + j < l; j++) {
340 if (ref != NULL && p[i + j] != ref[i + j])
342 fprintf(stderr, "%c%02x", sep, 0xff & (int)p[i+j]);
343 if (ref != NULL && p[i + j] == ref[i + j])
346 for (; j < 16; j++) {
347 fprintf(stderr, "%c ", sep);
350 fprintf(stderr, "%c", sep);
351 for (j=0; j < 16 && i + j < l; j++) {
353 if (c >= ' ' && c <= 126)
354 fprintf(stderr, "%c", c);
356 fprintf(stderr, ".");
358 fprintf(stderr, "\n");
362 /* assertEqualMem() displays the values of the two memory blocks. */
363 /* TODO: For long blocks, hexdump the first bytes that actually differ. */
365 test_assert_equal_mem(const char *file, int line,
366 const char *v1, const char *e1,
367 const char *v2, const char *e2,
368 size_t l, const char *ld, void *extra)
371 if (v1 == NULL || v2 == NULL) {
376 } else if (memcmp(v1, v2, l) == 0) {
381 if (previous_failures(file, line))
383 fprintf(stderr, "%s:%d: Assertion failed: memory not equal\n",
385 fprintf(stderr, " size %s = %d\n", ld, (int)l);
386 fprintf(stderr, " Dump of %s\n", e1);
387 hexdump(v1, v2, l < 32 ? l : 32, 0);
388 fprintf(stderr, " Dump of %s\n", e2);
389 hexdump(v2, v1, l < 32 ? l : 32, 0);
390 fprintf(stderr, "\n");
391 report_failure(extra);
396 test_assert_empty_file(const char *f1fmt, ...)
407 vsprintf(f1, f1fmt, ap);
410 if (stat(f1, &st) != 0) {
411 fprintf(stderr, "%s:%d: Could not stat: %s\n", test_filename, test_line, f1);
412 report_failure(NULL);
418 if (previous_failures(test_filename, test_line))
421 fprintf(stderr, "%s:%d: File not empty: %s\n", test_filename, test_line, f1);
422 fprintf(stderr, " File size: %d\n", (int)st.st_size);
423 fprintf(stderr, " Contents:\n");
424 fd = open(f1, O_RDONLY);
426 fprintf(stderr, " Unable to open %s\n", f1);
428 s = sizeof(buff) < st.st_size ? sizeof(buff) : st.st_size;
429 s = read(fd, buff, s);
430 hexdump(buff, NULL, s, 0);
432 report_failure(NULL);
436 /* assertEqualFile() asserts that two files have the same contents. */
437 /* TODO: hexdump the first bytes that actually differ. */
439 test_assert_equal_file(const char *f1, const char *f2pattern, ...)
448 va_start(ap, f2pattern);
449 vsprintf(f2, f2pattern, ap);
452 fd1 = open(f1, O_RDONLY);
453 fd2 = open(f2, O_RDONLY);
455 n1 = read(fd1, buff1, sizeof(buff1));
456 n2 = read(fd2, buff2, sizeof(buff2));
459 if (n1 == 0 && n2 == 0)
461 if (memcmp(buff1, buff2, n1) != 0)
465 if (previous_failures(test_filename, test_line))
467 fprintf(stderr, "%s:%d: Files are not identical\n",
468 test_filename, test_line);
469 fprintf(stderr, " file1=\"%s\"\n", f1);
470 fprintf(stderr, " file2=\"%s\"\n", f2);
471 report_failure(test_extra);
475 /* assertFileContents() asserts the contents of a file. */
477 test_assert_file_contents(const void *buff, int s, const char *fpattern, ...)
485 va_start(ap, fpattern);
486 vsprintf(f, fpattern, ap);
489 fd = open(f, O_RDONLY);
490 contents = malloc(s * 2);
491 n = read(fd, contents, s * 2);
492 if (n == s && memcmp(buff, contents, s) == 0) {
497 if (!previous_failures(test_filename, test_line)) {
498 fprintf(stderr, "%s:%d: File contents don't match\n",
499 test_filename, test_line);
500 fprintf(stderr, " file=\"%s\"\n", f);
502 hexdump(contents, buff, n, 0);
504 fprintf(stderr, " File empty, contents should be:\n");
505 hexdump(buff, NULL, s, 0);
507 report_failure(test_extra);
514 * Call standard system() call, but build up the command line using
515 * sprintf() conventions.
518 systemf(const char *fmt, ...)
525 vsprintf(buff, fmt, ap);
532 * Slurp a file into memory for ease of comparison and testing.
533 * Returns size of file in 'sizep' if non-NULL, null-terminates
534 * data in memory for ease of use.
537 slurpfile(size_t * sizep, const char *fmt, ...)
548 vsprintf(filename, fmt, ap);
551 fd = open(filename, O_RDONLY);
553 /* Note: No error; non-existent file is okay here. */
558 fprintf(stderr, "Can't stat file %s\n", filename);
562 p = malloc(st.st_size + 1);
564 fprintf(stderr, "Can't allocate %ld bytes of memory to read file %s\n", (long int)st.st_size, filename);
568 bytes_read = read(fd, p, st.st_size);
569 if (bytes_read < st.st_size) {
570 fprintf(stderr, "Can't read file %s\n", filename);
575 p[st.st_size] = '\0';
577 *sizep = (size_t)st.st_size;
583 * "list.h" is automatically generated; it just has a lot of lines like:
584 * DEFINE_TEST(function_name)
585 * It's used above to declare all of the test functions.
586 * We reuse it here to define a list of all tests (functions and names).
589 #define DEFINE_TEST(n) { n, #n },
590 struct { void (*func)(void); const char *name; } tests[] = {
595 * Each test is run in a private work dir. Those work dirs
596 * do have consistent and predictable names, in case a group
597 * of tests need to collaborate. However, there is no provision
598 * for requiring that tests run in a certain order.
600 static int test_run(int i, const char *tmpdir)
602 int failures_before = failures;
605 printf("%d: %s\n", i, tests[i].name);
607 * Always explicitly chdir() in case the last test moved us to
612 "ERROR: Couldn't chdir to temp dir %s\n",
616 /* Create a temp directory for this specific test. */
617 if (mkdir(tests[i].name, 0755)) {
619 "ERROR: Couldn't create temp dir ``%s''\n",
623 /* Chdir() to that work directory. */
624 if (chdir(tests[i].name)) {
626 "ERROR: Couldn't chdir to temp dir ``%s''\n",
630 /* Explicitly reset the locale before each test. */
631 setlocale(LC_ALL, "C");
632 /* Run the actual test. */
634 /* Summarize the results of this test. */
636 /* Return appropriate status. */
637 return (failures == failures_before ? 0 : 1);
640 static void usage(const char *program)
642 static const int limit = sizeof(tests) / sizeof(tests[0]);
645 printf("Usage: %s [options] <test> <test> ...\n", program);
646 printf("Default is to run all tests.\n");
647 printf("Otherwise, specify the numbers of the tests you wish to run.\n");
648 printf("Options:\n");
649 printf(" -k Keep running after failures.\n");
650 printf(" Default: Core dump after any failure.\n");
652 printf(" -p <path> Path to executable to be tested.\n");
653 printf(" Default: path taken from " ENVBASE " environment variable.\n");
655 printf(" -q Quiet.\n");
656 printf(" -r <dir> Path to dir containing reference files.\n");
657 printf(" Default: Current directory.\n");
658 printf("Available tests:\n");
659 for (i = 0; i < limit; i++)
660 printf(" %d: %s\n", i, tests[i].name);
664 #define UUDECODE(c) (((c) - 0x20) & 0x3f)
667 extract_reference_file(const char *name)
672 sprintf(buff, "%s/%s.uu", refdir, name);
673 in = fopen(buff, "r");
674 failure("Couldn't open reference file %s", buff);
678 /* Read up to and including the 'begin' line. */
680 if (fgets(buff, sizeof(buff), in) == NULL) {
681 /* TODO: This is a failure. */
684 if (memcmp(buff, "begin ", 6) == 0)
687 /* Now, decode the rest and write it. */
688 /* Not a lot of error checking here; the input better be right. */
689 out = fopen(name, "w");
690 while (fgets(buff, sizeof(buff), in) != NULL) {
694 if (memcmp(buff, "end", 3) == 0)
697 bytes = UUDECODE(*p++);
700 /* Write out 1-3 bytes from that. */
702 n = UUDECODE(*p++) << 18;
703 n |= UUDECODE(*p++) << 12;
708 n |= UUDECODE(*p++) << 6;
709 fputc((n >> 8) & 0xFF, out);
714 fputc(n & 0xFF, out);
724 int main(int argc, char **argv)
726 static const int limit = sizeof(tests) / sizeof(tests[0]);
727 int i, tests_run = 0, tests_failed = 0, opt;
729 char *refdir_alloc = NULL;
732 char tmpdir_timestamp[256];
735 * Name of this program, used to build root of our temp directory
738 progname = p = argv[0];
746 /* Get the target program from environment, if available. */
747 testprog = getenv(ENVBASE);
750 /* Allow -k to be controlled through the environment. */
751 if (getenv(ENVBASE "_KEEP_GOING") != NULL)
754 /* Get the directory holding test files from environment. */
755 refdir = getenv(ENVBASE "_TEST_FILES");
760 while ((opt = getopt(argc, argv, "kp:qr:")) != -1) {
787 * Sanity-check that our options make sense.
790 if (testprog == NULL)
795 * Create a temp directory for the following tests.
796 * Include the time the tests started as part of the name,
797 * to make it easier to track the results of multiple tests.
800 for (i = 0; i < 1000; i++) {
801 strftime(tmpdir_timestamp, sizeof(tmpdir_timestamp),
804 sprintf(tmpdir, "/tmp/%s.%s-%03d", progname, tmpdir_timestamp, i);
805 if (mkdir(tmpdir,0755) == 0)
809 fprintf(stderr, "ERROR: Unable to create temp directory %s\n",
815 * If the user didn't specify a directory for locating
816 * reference files, use the current directory for that.
818 if (refdir == NULL) {
819 systemf("/bin/pwd > %s/refdir", tmpdir);
820 refdir = refdir_alloc = slurpfile(NULL, "%s/refdir", tmpdir);
821 p = refdir + strlen(refdir);
822 while (p[-1] == '\n') {
829 * Banner with basic information.
832 printf("Running tests in: %s\n", tmpdir);
833 printf("Reference files will be read from: %s\n", refdir);
835 printf("Running tests on: %s\n", testprog);
837 printf("Exercising: ");
839 printf("%s\n", EXTRA_VERSION);
843 * Run some or all of the individual tests.
846 /* Default: Run all tests. */
847 for (i = 0; i < limit; i++) {
848 if (test_run(i, tmpdir))
853 while (*(argv) != NULL) {
855 if (**argv < '0' || **argv > '9' || i < 0 || i >= limit) {
856 printf("*** INVALID Test %s\n", *argv);
859 if (test_run(i, tmpdir))
868 * Report summary statistics.
872 printf("%d of %d tests reported failures\n",
873 tests_failed, tests_run);
874 printf(" Total of %d assertions checked.\n", assertions);
875 printf(" Total of %d assertions failed.\n", failures);
876 printf(" Total of %d assertions skipped.\n", skips);
881 return (tests_failed);