contrib/libevent/test/tinytest.c

   1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
   2  *
   3  * Redistribution and use in source and binary forms, with or without
   4  * modification, are permitted provided that the following conditions
   5  * are met:
   6  * 1. Redistributions of source code must retain the above copyright
   7  *    notice, this list of conditions and the following disclaimer.
   8  * 2. Redistributions in binary form must reproduce the above copyright
   9  *    notice, this list of conditions and the following disclaimer in the
  10  *    documentation and/or other materials provided with the distribution.
  11  * 3. The name of the author may not be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25 #ifdef TINYTEST_LOCAL
  26 #include "tinytest_local.h"
  27 #endif
  28
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <assert.h>
  33
  34 #ifndef NO_FORKING
  35
  36 #ifdef _WIN32
  37 #include <windows.h>
  38 #else
  39 #include <sys/types.h>
  40 #include <sys/wait.h>
  41 #include <unistd.h>
  42 #endif
  43
  44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
  45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
  46     __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
  47 /* Workaround for a stupid bug in OSX 10.6 */
  48 #define FORK_BREAKS_GCOV
  49 #include <vproc.h>
  50 #endif
  51 #endif
  52
  53 #endif /* !NO_FORKING */
  54
  55 #ifndef __GNUC__
  56 #define __attribute__(x)
  57 #endif
  58
  59 #include "tinytest.h"
  60 #include "tinytest_macros.h"
  61
  62 #define LONGEST_TEST_NAME 16384
  63 #define DEFAULT_TESTCASE_TIMEOUT 30U
  64 #define MAGIC_EXITCODE 42
  65
  66 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
  67 static int n_ok = 0; /**< Number of tests that have passed */
  68 static int n_bad = 0; /**< Number of tests that have failed. */
  69 static int n_skipped = 0; /**< Number of tests that have been skipped. */
  70
  71 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
  72 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
  73 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
  74 static unsigned int opt_timeout = DEFAULT_TESTCASE_TIMEOUT; /**< Timeout for every test (using alarm()) */
  75 const char *verbosity_flag = "";
  76
  77 const struct testlist_alias_t *cfg_aliases=NULL;
  78
  79 enum outcome { SKIP=2, OK=1, FAIL=0 };
  80 static enum outcome cur_test_outcome = 0;
  81 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
  82 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
  83 const char *cur_test_name = NULL;
  84
  85 static void usage(struct testgroup_t *groups, int list_groups)
  86         __attribute__((noreturn));
  87 static int process_test_option(struct testgroup_t *groups, const char *test);
  88
  89 #ifdef _WIN32
  90 /* Copy of argv[0] for win32. */
  91 static char commandname[MAX_PATH+1];
  92
  93 struct timeout_thread_args {
  94         const testcase_fn *fn;
  95         void *env;
  96 };
  97
  98 static DWORD WINAPI
  99 timeout_thread_proc_(LPVOID arg)
 100 {
 101         struct timeout_thread_args *args = arg;
 102         (*(args->fn))(args->env);
 103         ExitThread(cur_test_outcome == FAIL ? 1 : 0);
 104 }
 105
 106 static enum outcome
 107 testcase_run_in_thread_(const struct testcase_t *testcase, void *env)
 108 {
 109         /* We will never run testcase in a new thread when the
 110         timeout is set to zero */
 111         assert(opt_timeout);
 112         DWORD ret, tid;
 113         HANDLE handle;
 114         struct timeout_thread_args args = {
 115                 &(testcase->fn),
 116                 env
 117         };
 118
 119         handle =CreateThread(NULL, 0, timeout_thread_proc_,
 120                 (LPVOID)&args, 0, &tid);
 121         ret = WaitForSingleObject(handle, opt_timeout * 1000U);
 122         if (ret == WAIT_OBJECT_0) {
 123                 ret = 0;
 124                 if (!GetExitCodeThread(handle, &ret)) {
 125                         printf("GetExitCodeThread failed\n");
 126                         ret = 1;
 127                 }
 128         } else if (ret == WAIT_TIMEOUT) {
 129                 printf("timeout\n");
 130         } else {
 131                 printf("Wait failed\n");
 132         }
 133         CloseHandle(handle);
 134         if (ret == 0)
 135                 return OK;
 136         else if (ret == MAGIC_EXITCODE)
 137                 return SKIP;
 138         else
 139                 return FAIL;
 140 }
 141 #else
 142 static unsigned int testcase_set_timeout_(void)
 143 {
 144         return alarm(opt_timeout);
 145 }
 146
 147 static unsigned int testcase_reset_timeout_(void)
 148 {
 149         return alarm(0);
 150 }
 151 #endif
 152
 153 static enum outcome
 154 testcase_run_bare_(const struct testcase_t *testcase)
 155 {
 156         void *env = NULL;
 157         int outcome;
 158         if (testcase->setup) {
 159                 env = testcase->setup->setup_fn(testcase);
 160                 if (!env)
 161                         return FAIL;
 162                 else if (env == (void*)TT_SKIP)
 163                         return SKIP;
 164         }
 165
 166         cur_test_outcome = OK;
 167         {
 168                 if (opt_timeout) {
 169 #ifdef _WIN32
 170                         cur_test_outcome = testcase_run_in_thread_(testcase, env);
 171 #else
 172                         testcase_set_timeout_();
 173                         testcase->fn(env);
 174                         testcase_reset_timeout_();
 175 #endif
 176                 } else {
 177                         testcase->fn(env);
 178                 }
 179         }
 180         outcome = cur_test_outcome;
 181
 182         if (testcase->setup) {
 183                 if (testcase->setup->cleanup_fn(testcase, env) == 0)
 184                         outcome = FAIL;
 185         }
 186
 187         return outcome;
 188 }
 189
 190
 191 #ifndef NO_FORKING
 192
 193 static enum outcome
 194 testcase_run_forked_(const struct testgroup_t *group,
 195                      const struct testcase_t *testcase)
 196 {
 197 #ifdef _WIN32
 198         /* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
 199            we'll invoke our own exe (whose name we recall from the command
 200            line) with a command line that tells it to run just the test we
 201            want, and this time without forking.
 202
 203            (No, threads aren't an option.  The whole point of forking is to
 204            share no state between tests.)
 205          */
 206         int ok;
 207         char buffer[LONGEST_TEST_NAME+256];
 208         STARTUPINFOA si;
 209         PROCESS_INFORMATION info;
 210         DWORD ret;
 211
 212         if (!in_tinytest_main) {
 213                 printf("\nERROR.  On Windows, testcase_run_forked_ must be"
 214                        " called from within tinytest_main.\n");
 215                 abort();
 216         }
 217         if (opt_verbosity>0)
 218                 printf("[forking] ");
 219
 220         snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s --timeout 0 %s%s",
 221                  commandname, verbosity_flag, group->prefix, testcase->name);
 222
 223         memset(&si, 0, sizeof(si));
 224         memset(&info, 0, sizeof(info));
 225         si.cb = sizeof(si);
 226
 227         ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
 228                            0, NULL, NULL, &si, &info);
 229         if (!ok) {
 230                 printf("CreateProcess failed!\n");
 231                 return FAIL;
 232         }
 233         ret = WaitForSingleObject(info.hProcess,
 234                 (opt_timeout ? opt_timeout * 1000U : INFINITE));
 235
 236         if (ret == WAIT_OBJECT_0) {
 237                 GetExitCodeProcess(info.hProcess, &ret);
 238         } else if (ret == WAIT_TIMEOUT) {
 239                 printf("timeout\n");
 240         } else {
 241                 printf("Wait failed\n");
 242         }
 243         CloseHandle(info.hProcess);
 244         CloseHandle(info.hThread);
 245         if (ret == 0)
 246                 return OK;
 247         else if (ret == MAGIC_EXITCODE)
 248                 return SKIP;
 249         else
 250                 return FAIL;
 251 #else
 252         int outcome_pipe[2];
 253         pid_t pid;
 254         (void)group;
 255
 256         if (pipe(outcome_pipe))
 257                 perror("opening pipe");
 258
 259         if (opt_verbosity>0)
 260                 printf("[forking] ");
 261         pid = fork();
 262 #ifdef FORK_BREAKS_GCOV
 263         vproc_transaction_begin(0);
 264 #endif
 265         if (!pid) {
 266                 /* child. */
 267                 int test_r, write_r;
 268                 char b[1];
 269                 close(outcome_pipe[0]);
 270                 test_r = testcase_run_bare_(testcase);
 271                 assert(0<=(int)test_r && (int)test_r<=2);
 272                 b[0] = "NYS"[test_r];
 273                 write_r = (int)write(outcome_pipe[1], b, 1);
 274                 if (write_r != 1) {
 275                         perror("write outcome to pipe");
 276                         exit(1);
 277                 }
 278                 exit(0);
 279                 return FAIL; /* unreachable */
 280         } else {
 281                 /* parent */
 282                 int status, r, exitcode;
 283                 char b[1];
 284                 /* Close this now, so that if the other side closes it,
 285                  * our read fails. */
 286                 close(outcome_pipe[1]);
 287                 r = (int)read(outcome_pipe[0], b, 1);
 288                 if (r == 0) {
 289                         printf("[Lost connection!] ");
 290                         return FAIL;
 291                 } else if (r != 1) {
 292                         perror("read outcome from pipe");
 293                 }
 294                 waitpid(pid, &status, 0);
 295                 exitcode = WEXITSTATUS(status);
 296                 close(outcome_pipe[0]);
 297                 if (opt_verbosity>1)
 298                         printf("%s%s: exited with %i (%i)\n", group->prefix, testcase->name, exitcode, status);
 299                 if (exitcode != 0)
 300                 {
 301                         printf("[atexit failure!] ");
 302                         return FAIL;
 303                 }
 304                 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
 305         }
 306 #endif
 307 }
 308
 309 #endif /* !NO_FORKING */
 310
 311 int
 312 testcase_run_one(const struct testgroup_t *group,
 313                  const struct testcase_t *testcase)
 314 {
 315         enum outcome outcome;
 316
 317         if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
 318                 if (opt_verbosity>0)
 319                         printf("%s%s: %s\n",
 320                            group->prefix, testcase->name,
 321                            (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
 322                 ++n_skipped;
 323                 return SKIP;
 324         }
 325
 326         if (opt_verbosity>0 && !opt_forked) {
 327                 printf("%s%s: ", group->prefix, testcase->name);
 328         } else {
 329                 if (opt_verbosity==0) printf(".");
 330                 cur_test_prefix = group->prefix;
 331                 cur_test_name = testcase->name;
 332         }
 333
 334 #ifndef NO_FORKING
 335         if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
 336                 outcome = testcase_run_forked_(group, testcase);
 337         } else {
 338 #else
 339         {
 340 #endif
 341                 outcome = testcase_run_bare_(testcase);
 342         }
 343
 344         if (outcome == OK) {
 345                 if (opt_verbosity>0 && !opt_forked)
 346                         puts(opt_verbosity==1?"OK":"");
 347         } else if (outcome == SKIP) {
 348                 if (opt_verbosity>0 && !opt_forked)
 349                         puts("SKIPPED");
 350         } else {
 351                 if (!opt_forked)
 352                         printf("\n  [%s FAILED]\n", testcase->name);
 353         }
 354
 355         if (opt_forked) {
 356                 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
 357                 return 1; /* unreachable */
 358         } else {
 359                 return (int)outcome;
 360         }
 361 }
 362
 363 int
 364 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
 365 {
 366         int i, j;
 367         size_t length = LONGEST_TEST_NAME;
 368         char fullname[LONGEST_TEST_NAME];
 369         int found=0;
 370         if (strstr(arg, ".."))
 371                 length = strstr(arg,"..")-arg;
 372         for (i=0; groups[i].prefix; ++i) {
 373                 for (j=0; groups[i].cases[j].name; ++j) {
 374                         struct testcase_t *testcase = &groups[i].cases[j];
 375                         snprintf(fullname, sizeof(fullname), "%s%s",
 376                                  groups[i].prefix, testcase->name);
 377                         if (!flag) { /* Hack! */
 378                                 printf("    %s", fullname);
 379                                 if (testcase->flags & TT_OFF_BY_DEFAULT)
 380                                         puts("   (Off by default)");
 381                                 else if (testcase->flags & TT_SKIP)
 382                                         puts("  (DISABLED)");
 383                                 else
 384                                         puts("");
 385                         }
 386                         if (!strncmp(fullname, arg, length)) {
 387                                 if (set)
 388                                         testcase->flags |= flag;
 389                                 else
 390                                         testcase->flags &= ~flag;
 391                                 ++found;
 392                         }
 393                 }
 394         }
 395         return found;
 396 }
 397
 398 static void
 399 usage(struct testgroup_t *groups, int list_groups)
 400 {
 401         puts("Options are: [--verbose|--quiet|--terse] [--no-fork] [--timeout <sec>]");
 402         puts("  Specify tests by name, or using a prefix ending with '..'");
 403         puts("  To skip a test, prefix its name with a colon.");
 404         puts("  To enable a disabled test, prefix its name with a plus.");
 405         puts("  Use --list-tests for a list of tests.");
 406         if (list_groups) {
 407                 puts("Known tests are:");
 408                 tinytest_set_flag_(groups, "..", 1, 0);
 409         }
 410         exit(0);
 411 }
 412
 413 static int
 414 process_test_alias(struct testgroup_t *groups, const char *test)
 415 {
 416         int i, j, n, r;
 417         for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
 418                 if (!strcmp(cfg_aliases[i].name, test)) {
 419                         n = 0;
 420                         for (j = 0; cfg_aliases[i].tests[j]; ++j) {
 421                                 r = process_test_option(groups, cfg_aliases[i].tests[j]);
 422                                 if (r<0)
 423                                         return -1;
 424                                 n += r;
 425                         }
 426                         return n;
 427                 }
 428         }
 429         printf("No such test alias as @%s!",test);
 430         return -1;
 431 }
 432
 433 static int
 434 process_test_option(struct testgroup_t *groups, const char *test)
 435 {
 436         int flag = TT_ENABLED_;
 437         int n = 0;
 438         if (test[0] == '@') {
 439                 return process_test_alias(groups, test + 1);
 440         } else if (test[0] == ':') {
 441                 ++test;
 442                 flag = TT_SKIP;
 443         } else if (test[0] == '+') {
 444                 ++test;
 445                 ++n;
 446                 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
 447                         printf("No such test as %s!\n", test);
 448                         return -1;
 449                 }
 450         } else {
 451                 ++n;
 452         }
 453         if (!tinytest_set_flag_(groups, test, 1, flag)) {
 454                 printf("No such test as %s!\n", test);
 455                 return -1;
 456         }
 457         return n;
 458 }
 459
 460 void
 461 tinytest_set_aliases(const struct testlist_alias_t *aliases)
 462 {
 463         cfg_aliases = aliases;
 464 }
 465
 466 int
 467 tinytest_main(int c, const char **v, struct testgroup_t *groups)
 468 {
 469         int i, j, n=0;
 470
 471 #ifdef _WIN32
 472         const char *sp = strrchr(v[0], '.');
 473         const char *extension = "";
 474         if (!sp || stricmp(sp, ".exe"))
 475                 extension = ".exe"; /* Add an exe so CreateProcess will work */
 476         snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
 477         commandname[MAX_PATH]='\0';
 478 #endif
 479         for (i=1; i<c; ++i) {
 480                 if (v[i][0] == '-') {
 481                         if (!strcmp(v[i], "--RUNNING-FORKED")) {
 482                                 opt_forked = 1;
 483                         } else if (!strcmp(v[i], "--no-fork")) {
 484                                 opt_nofork = 1;
 485                         } else if (!strcmp(v[i], "--quiet")) {
 486                                 opt_verbosity = -1;
 487                                 verbosity_flag = "--quiet";
 488                         } else if (!strcmp(v[i], "--verbose")) {
 489                                 opt_verbosity = 2;
 490                                 verbosity_flag = "--verbose";
 491                         } else if (!strcmp(v[i], "--terse")) {
 492                                 opt_verbosity = 0;
 493                                 verbosity_flag = "--terse";
 494                         } else if (!strcmp(v[i], "--help")) {
 495                                 usage(groups, 0);
 496                         } else if (!strcmp(v[i], "--list-tests")) {
 497                                 usage(groups, 1);
 498                         } else if (!strcmp(v[i], "--timeout")) {
 499                                 ++i;
 500                                 if (i >= c) {
 501                                         fprintf(stderr, "--timeout requires argument\n");
 502                                         return -1;
 503                                 }
 504                                 opt_timeout = (unsigned)atoi(v[i]);
 505                         } else {
 506                                 fprintf(stderr, "Unknown option %s. Try --help\n", v[i]);
 507                                 return -1;
 508                         }
 509                 } else {
 510                         int r = process_test_option(groups, v[i]);
 511                         if (r<0)
 512                                 return -1;
 513                         n += r;
 514                 }
 515         }
 516         if (!n)
 517                 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
 518
 519 #ifdef _IONBF
 520         setvbuf(stdout, NULL, _IONBF, 0);
 521 #endif
 522
 523         ++in_tinytest_main;
 524         for (i = 0; groups[i].prefix; ++i) {
 525                 struct testgroup_t *group = &groups[i];
 526                 for (j = 0; group->cases[j].name; ++j) {
 527                         struct testcase_t *testcase = &group->cases[j];
 528                         int test_attempts = 3;
 529                         int test_ret_err;
 530
 531                         if (!(testcase->flags & TT_ENABLED_))
 532                                 continue;
 533
 534                         for (;;) {
 535                                 test_ret_err = testcase_run_one(group, testcase);
 536
 537                                 if (test_ret_err == OK)
 538                                         break;
 539                                 if (!(testcase->flags & TT_RETRIABLE))
 540                                         break;
 541                                 printf("\n  [RETRYING %s (%i)]\n", testcase->name, test_attempts);
 542                                 if (!test_attempts--)
 543                                         break;
 544                         }
 545
 546                         switch (test_ret_err) {
 547                                 case OK:   ++n_ok;      break;
 548                                 case SKIP: ++n_skipped; break;
 549                                 default:   ++n_bad;     break;
 550                         }
 551                 }
 552         }
 553
 554         --in_tinytest_main;
 555
 556         if (opt_verbosity==0)
 557                 puts("");
 558
 559         if (n_bad)
 560                 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
 561                        n_bad+n_ok,n_skipped);
 562         else if (opt_verbosity >= 1)
 563                 printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
 564
 565         return (n_bad == 0) ? 0 : 1;
 566 }
 567
 568 int
 569 tinytest_get_verbosity_(void)
 570 {
 571         return opt_verbosity;
 572 }
 573
 574 void
 575 tinytest_set_test_failed_(void)
 576 {
 577         if (opt_verbosity <= 0 && cur_test_name) {
 578                 if (opt_verbosity==0) puts("");
 579                 printf("%s%s: ", cur_test_prefix, cur_test_name);
 580                 cur_test_name = NULL;
 581         }
 582         cur_test_outcome = FAIL;
 583 }
 584
 585 void
 586 tinytest_set_test_skipped_(void)
 587 {
 588         if (cur_test_outcome==OK)
 589                 cur_test_outcome = SKIP;
 590 }
 591
 592 char *
 593 tinytest_format_hex_(const void *val_, unsigned long len)
 594 {
 595         const unsigned char *val = val_;
 596         char *result, *cp;
 597         size_t i;
 598
 599         if (!val)
 600                 return strdup("null");
 601         if (!(result = malloc(len*2+1)))
 602                 return strdup("<allocation failure>");
 603         cp = result;
 604         for (i=0;i<len;++i) {
 605                 *cp++ = "0123456789ABCDEF"[val[i] >> 4];
 606                 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
 607         }
 608         *cp = 0;
 609         return result;
 610 }