3 #############################################################################
5 # Copyright (C) 1999-2001 Jason Evans <jasone@freebsd.org>.
8 # Redistribution and use in source and binary forms, with or without
9 # modification, are permitted provided that the following conditions
11 # 1. Redistributions of source code must retain the above copyright
12 # notice(s), this list of conditions and the following disclaimer as
13 # the first lines of this file unmodified other than the possible
14 # addition of one or more copyright notices.
15 # 2. Redistributions in binary form must reproduce the above copyright
16 # notice(s), this list of conditions and the following disclaimer in
17 # the documentation and/or other materials provided with the
20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
21 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
24 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
27 # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
28 # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
29 # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
30 # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 #############################################################################
38 #############################################################################
45 # Parse command-line arguments.
48 Getopt::Long::config("bundling"); # Allow -hv rather than forcing -h -v.
50 # Set option defaults for optional arguments.
60 &GetOptions("h|help" => \$opt_help,
61 "v|verbose" => \$opt_verbose,
62 "q|quiet" => \$opt_quiet,
63 "s|srcdir=s" => \$opt_srcdir,
64 "o|objdir=s" => \$opt_objdir,
65 "u|ustats" => \$opt_ustats,
66 "z|zero" => \$opt_zero
81 if ($opt_verbose && $opt_quiet)
83 print STDERR "-v and -q are incompatible\n";
90 print STDERR "No tests specified\n";
97 print STDERR "Option values: h:$opt_help, v:$opt_verbose, "
98 . "s:\"$opt_srcdir\", o:\"$opt_objdir\" "
99 . "q:$opt_quiet, u:$opt_ustats, z:$opt_zero\n";
100 printf STDERR "Tests (%d total): @ARGV\n", $#ARGV + 1;
104 # Create and print header.
108 "--------------------------------------------------------------------------\n",
109 "Test c_user c_system c_total chng\n",
110 " passed/FAILED h_user h_system h_total %% chng\n"
115 foreach $line (@TSTATS)
117 printf STDOUT "$line";
122 # Run sequence test(s).
124 $total_utime = 0.0; # Total user time.
125 $total_stime = 0.0; # Total system time.
126 $total_hutime = 0.0; # Total historical user time.
127 $total_hstime = 0.0; # Total historical system time.
128 $total_ntime = 0.0; # Total time for tests that have historical data.
130 foreach $test (@ARGV)
132 # Strip out any whitespace in $test.
133 $test =~ s/^\s*(.*)\s*$/$1/;
137 if (-e "$opt_srcdir/$test.exp")
141 ($okay, $utime, $stime) = &run_test($test);
143 if (-e "$opt_objdir/$test.out")
145 `diff $opt_srcdir/$test.exp $opt_objdir/$test.out > $opt_objdir/$test.diff 2>&1`;
148 # diff returns non-zero if there is a difference.
158 "Nonexistent output file \"$opt_objdir/$test.out\"\n";
162 ($hutime, $hstime) = &print_stats($test, $okay, 0, 0, $utime, $stime);
168 ($okay, $utime, $stime) = &run_test($test);
170 if (open (STEST_OUT, "<$opt_objdir/$test.out"))
173 $num_failed_subtests = 0;
175 while (defined($line = <STEST_OUT>))
177 if ($line =~ /1\.\.(\d+)/)
183 if ($num_subtests == 0)
188 print STDERR "Malformed or missing 1..n line\n";
193 for ($subtest = 1; $subtest <= $num_subtests; $subtest++)
195 while (defined($line = <STEST_OUT>))
197 if ($line =~ /^not\s+ok\s+(\d+)?/)
203 elsif ($line =~ /^ok\s+(\d+)?/)
212 if (defined($test_num) && ($test_num != $subtest))
214 # There was no output printed for one or more tests.
215 for (; $subtest < $test_num; $subtest++)
217 $num_failed_subtests++;
222 $num_failed_subtests++;
227 for (; $subtest <= $num_subtests; $subtest++)
229 $num_failed_subtests++;
234 if (0 < $num_failed_subtests)
244 print STDERR "Cannot open output file \"$opt_objdir/$test.out\"\n";
249 ($hutime, $hstime) = &print_stats($test, $okay,
250 $num_failed_subtests, $num_subtests,
254 $total_hutime += $hutime;
255 $total_hstime += $hstime;
259 $total_utime += $utime;
260 $total_stime += $stime;
264 @FAILED_TESTS = (@FAILED_TESTS, $test);
267 # If there were historical data, add the run time to the total time to
268 # compare against the historical run time.
269 if (0 < ($hutime + $hstime))
271 $total_ntime += $utime + $stime;
275 # Print summary stats.
276 $tt_str = sprintf ("%d / %d passed (%5.2f%%%%)",
277 ($#ARGV + 1) - ($#FAILED_TESTS + 1),
279 (($#ARGV + 1) - ($#FAILED_TESTS + 1))
280 / ($#ARGV + 1) * 100);
282 $t_str = sprintf ("Totals %7.2f %7.2f %7.2f"
284 . " %s %7.2f %7.2f %7.2f %7.2f%%%%\n",
285 $total_utime, $total_stime, $total_utime + $total_stime,
286 ($total_ntime - ($total_hutime + $total_hstime)),
287 $tt_str . ' ' x (40 - length($tt_str)),
288 $total_hutime, $total_hstime, $total_hutime + $total_hstime,
289 ($total_hutime + $total_hstime == 0.0) ? 0.0 :
291 - ($total_hutime + $total_hstime))
292 / ($total_hutime + $total_hstime) * 100));
294 @TSTATS = ("--------------------------------------------------------------------------\n",
296 "--------------------------------------------------------------------------\n"
300 foreach $line (@TSTATS)
302 printf STDOUT "$line";
306 if ($#FAILED_TESTS >= 0)
308 # One or more tests failed, so return an error.
311 # End of main execution.
317 my ($tutime, $tstime);
318 my ($utime, $stime, $cutime, $cstime);
319 my (@TSTATS, @TPATH);
321 my ($srcdir, $objdir);
323 # Get the path component of $test, if any.
324 @TPATH = split(/\//, $test);
326 $srcdir = join('/', ($opt_srcdir, @TPATH));
327 $objdir = join('/', ($opt_objdir, @TPATH));
329 @TSTATS = ("--------------------------------------------------------------------------\n");
331 $t_str = sprintf ("%s%s", $test, ' ' x (40 - length($test)));
332 @TSTATS = (@TSTATS, $t_str);
333 @STATS = (@STATS, @TSTATS);
336 foreach $line (@TSTATS)
338 printf STDOUT "$line";
342 ($utime, $stime, $cutime, $cstime) = times;
343 `$opt_objdir/$test $srcdir $objdir > $opt_objdir/$test.out 2>&1`;
344 ($utime, $stime, $tutime, $tstime) = times;
346 # Subtract the before time from the after time.
358 "\"$opt_objdir/$test > $opt_objdir/$test.out 2>&1\" returned $?\n";
363 return ($okay, $tutime, $tstime);
368 my ($test, $okay, $failed_subtests, $subtests, $utime, $stime) = @_;
369 my ($hutime, $hstime);
372 my ($t_str, $pass_str);
374 $pass_str = $okay ? "passed" : "*** FAILED ***";
375 if ((0 != $subtests) && (!$okay))
377 $pass_str = $pass_str . " ($failed_subtests/$subtests failed)";
379 $pass_str = $pass_str . ' ' x (39 - length($pass_str));
383 if (!open (TEST_PERF, "<$opt_objdir/$test.perf"))
385 print STDERR "Unable to open \"$opt_objdir/$test.perf\"\n";
390 ($hutime, $hstime) = split;
393 $t_str = sprintf (" %7.2f %7.2f %7.2f %7.2f\n"
394 . " %s %7.2f %7.2f %7.2f %7.2f%%%%\n",
395 $utime, $stime, $utime + $stime,
396 ($utime + $stime) - ($hutime + $hstime),
398 $hutime, $hstime, $hutime + $hstime,
399 (($hutime + $hstime) == 0.0) ? 0.0 :
400 ((($utime + $stime) - ($hutime + $hstime))
401 / ($hutime + $hstime) * 100));
408 $t_str = sprintf (" %7.2f %7.2f %7.2f \n"
410 $utime, $stime, $utime + $stime,
416 foreach $line (@TSTATS)
418 printf STDOUT "$line";
422 if ($okay && $opt_ustats)
424 if (!open (TEST_PERF, ">$opt_objdir/$test.perf"))
428 print STDERR "Unable to update \"$opt_objdir/$test.perf\"\n";
433 print TEST_PERF "$utime $stime\n";
438 return ($hutime, $hstime);
445 $0 [<options>] <test>+
448 --------------+-------------------------------------------------------------
449 -h --help | Print usage and exit.
450 -v --verbose | Verbose (incompatible with quiet).
451 -q --quiet | Quiet (incompatible with verbose).
452 -s --srcdir | Path to source tree (default is ".").
453 -o --objdir | Path to object tree (default is ".").
454 -u --ustats | Update historical statistics (stored in "<test>.perf".
455 -z --zero | Consider non-zero exit code to be an error.
456 --------------+-------------------------------------------------------------
458 If <test>.exp exists, <test>'s output is diff'ed with <test>.exp. Any
459 difference is considered failure.
461 If <test>.exp does not exist, output to stdout of the following form is
472 Lines which do not match the patterns shown above are ignored.