contrib/ntp/sntp/libevent/test/tinytest.c

   1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
   2  *
   3  * Redistribution and use in source and binary forms, with or without
   4  * modification, are permitted provided that the following conditions
   5  * are met:
   6  * 1. Redistributions of source code must retain the above copyright
   7  *    notice, this list of conditions and the following disclaimer.
   8  * 2. Redistributions in binary form must reproduce the above copyright
   9  *    notice, this list of conditions and the following disclaimer in the
  10  *    documentation and/or other materials provided with the distribution.
  11  * 3. The name of the author may not be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25 #ifdef TINYTEST_LOCAL
  26 #include "tinytest_local.h"
  27 #endif
  28
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <assert.h>
  33
  34 #ifndef NO_FORKING
  35
  36 #ifdef _WIN32
  37 #include <windows.h>
  38 #else
  39 #include <sys/types.h>
  40 #include <sys/wait.h>
  41 #include <unistd.h>
  42 #endif
  43
  44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
  45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
  46     __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
  47 /* Workaround for a stupid bug in OSX 10.6 */
  48 #define FORK_BREAKS_GCOV
  49 #include <vproc.h>
  50 #endif
  51 #endif
  52
  53 #endif /* !NO_FORKING */
  54
  55 #ifndef __GNUC__
  56 #define __attribute__(x)
  57 #endif
  58
  59 #include "tinytest.h"
  60 #include "tinytest_macros.h"
  61
  62 #define LONGEST_TEST_NAME 16384
  63
  64 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
  65 static int n_ok = 0; /**< Number of tests that have passed */
  66 static int n_bad = 0; /**< Number of tests that have failed. */
  67 static int n_skipped = 0; /**< Number of tests that have been skipped. */
  68
  69 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
  70 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
  71 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
  72 const char *verbosity_flag = "";
  73
  74 const struct testlist_alias_t *cfg_aliases=NULL;
  75
  76 enum outcome { SKIP=2, OK=1, FAIL=0 };
  77 static enum outcome cur_test_outcome = 0;
  78 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
  79 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
  80 const char *cur_test_name = NULL;
  81
  82 #ifdef _WIN32
  83 /* Copy of argv[0] for win32. */
  84 static char commandname[MAX_PATH+1];
  85 #endif
  86
  87 static void usage(struct testgroup_t *groups, int list_groups)
  88   __attribute__((noreturn));
  89 static int process_test_option(struct testgroup_t *groups, const char *test);
  90
  91 static enum outcome
  92 testcase_run_bare_(const struct testcase_t *testcase)
  93 {
  94         void *env = NULL;
  95         int outcome;
  96         if (testcase->setup) {
  97                 env = testcase->setup->setup_fn(testcase);
  98                 if (!env)
  99                         return FAIL;
 100                 else if (env == (void*)TT_SKIP)
 101                         return SKIP;
 102         }
 103
 104         cur_test_outcome = OK;
 105         testcase->fn(env);
 106         outcome = cur_test_outcome;
 107
 108         if (testcase->setup) {
 109                 if (testcase->setup->cleanup_fn(testcase, env) == 0)
 110                         outcome = FAIL;
 111         }
 112
 113         return outcome;
 114 }
 115
 116 #define MAGIC_EXITCODE 42
 117
 118 #ifndef NO_FORKING
 119
 120 static enum outcome
 121 testcase_run_forked_(const struct testgroup_t *group,
 122                      const struct testcase_t *testcase)
 123 {
 124 #ifdef _WIN32
 125         /* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
 126            we'll invoke our own exe (whose name we recall from the command
 127            line) with a command line that tells it to run just the test we
 128            want, and this time without forking.
 129
 130            (No, threads aren't an option.  The whole point of forking is to
 131            share no state between tests.)
 132          */
 133         int ok;
 134         char buffer[LONGEST_TEST_NAME+256];
 135         STARTUPINFOA si;
 136         PROCESS_INFORMATION info;
 137         DWORD exitcode;
 138
 139         if (!in_tinytest_main) {
 140                 printf("\nERROR.  On Windows, testcase_run_forked_ must be"
 141                        " called from within tinytest_main.\n");
 142                 abort();
 143         }
 144         if (opt_verbosity>0)
 145                 printf("[forking] ");
 146
 147         snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
 148                  commandname, verbosity_flag, group->prefix, testcase->name);
 149
 150         memset(&si, 0, sizeof(si));
 151         memset(&info, 0, sizeof(info));
 152         si.cb = sizeof(si);
 153
 154         ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
 155                            0, NULL, NULL, &si, &info);
 156         if (!ok) {
 157                 printf("CreateProcess failed!\n");
 158                 return 0;
 159         }
 160         WaitForSingleObject(info.hProcess, INFINITE);
 161         GetExitCodeProcess(info.hProcess, &exitcode);
 162         CloseHandle(info.hProcess);
 163         CloseHandle(info.hThread);
 164         if (exitcode == 0)
 165                 return OK;
 166         else if (exitcode == MAGIC_EXITCODE)
 167                 return SKIP;
 168         else
 169                 return FAIL;
 170 #else
 171         int outcome_pipe[2];
 172         pid_t pid;
 173         (void)group;
 174
 175         if (pipe(outcome_pipe))
 176                 perror("opening pipe");
 177
 178         if (opt_verbosity>0)
 179                 printf("[forking] ");
 180         pid = fork();
 181 #ifdef FORK_BREAKS_GCOV
 182         vproc_transaction_begin(0);
 183 #endif
 184         if (!pid) {
 185                 /* child. */
 186                 int test_r, write_r;
 187                 char b[1];
 188                 close(outcome_pipe[0]);
 189                 test_r = testcase_run_bare_(testcase);
 190                 assert(0<=(int)test_r && (int)test_r<=2);
 191                 b[0] = "NYS"[test_r];
 192                 write_r = (int)write(outcome_pipe[1], b, 1);
 193                 if (write_r != 1) {
 194                         perror("write outcome to pipe");
 195                         exit(1);
 196                 }
 197                 exit(0);
 198                 return FAIL; /* unreachable */
 199         } else {
 200                 /* parent */
 201                 int status, r;
 202                 char b[1];
 203                 /* Close this now, so that if the other side closes it,
 204                  * our read fails. */
 205                 close(outcome_pipe[1]);
 206                 r = (int)read(outcome_pipe[0], b, 1);
 207                 if (r == 0) {
 208                         printf("[Lost connection!] ");
 209                         return 0;
 210                 } else if (r != 1) {
 211                         perror("read outcome from pipe");
 212                 }
 213                 waitpid(pid, &status, 0);
 214                 close(outcome_pipe[0]);
 215                 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
 216         }
 217 #endif
 218 }
 219
 220 #endif /* !NO_FORKING */
 221
 222 int
 223 testcase_run_one(const struct testgroup_t *group,
 224                  const struct testcase_t *testcase)
 225 {
 226         enum outcome outcome;
 227
 228         if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
 229                 if (opt_verbosity>0)
 230                         printf("%s%s: %s\n",
 231                            group->prefix, testcase->name,
 232                            (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
 233                 ++n_skipped;
 234                 return SKIP;
 235         }
 236
 237         if (opt_verbosity>0 && !opt_forked) {
 238                 printf("%s%s: ", group->prefix, testcase->name);
 239         } else {
 240                 if (opt_verbosity==0) printf(".");
 241                 cur_test_prefix = group->prefix;
 242                 cur_test_name = testcase->name;
 243         }
 244
 245 #ifndef NO_FORKING
 246         if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
 247                 outcome = testcase_run_forked_(group, testcase);
 248         } else {
 249 #else
 250         {
 251 #endif
 252                 outcome = testcase_run_bare_(testcase);
 253         }
 254
 255         if (outcome == OK) {
 256                 ++n_ok;
 257                 if (opt_verbosity>0 && !opt_forked)
 258                         puts(opt_verbosity==1?"OK":"");
 259         } else if (outcome == SKIP) {
 260                 ++n_skipped;
 261                 if (opt_verbosity>0 && !opt_forked)
 262                         puts("SKIPPED");
 263         } else {
 264                 ++n_bad;
 265                 if (!opt_forked)
 266                         printf("\n  [%s FAILED]\n", testcase->name);
 267         }
 268
 269         if (opt_forked) {
 270                 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
 271                 return 1; /* unreachable */
 272         } else {
 273                 return (int)outcome;
 274         }
 275 }
 276
 277 int
 278 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
 279 {
 280         int i, j;
 281         size_t length = LONGEST_TEST_NAME;
 282         char fullname[LONGEST_TEST_NAME];
 283         int found=0;
 284         if (strstr(arg, ".."))
 285                 length = strstr(arg,"..")-arg;
 286         for (i=0; groups[i].prefix; ++i) {
 287                 for (j=0; groups[i].cases[j].name; ++j) {
 288                         struct testcase_t *testcase = &groups[i].cases[j];
 289                         snprintf(fullname, sizeof(fullname), "%s%s",
 290                                  groups[i].prefix, testcase->name);
 291                         if (!flag) { /* Hack! */
 292                                 printf("    %s", fullname);
 293                                 if (testcase->flags & TT_OFF_BY_DEFAULT)
 294                                         puts("   (Off by default)");
 295                                 else if (testcase->flags & TT_SKIP)
 296                                         puts("  (DISABLED)");
 297                                 else
 298                                         puts("");
 299                         }
 300                         if (!strncmp(fullname, arg, length)) {
 301                                 if (set)
 302                                         testcase->flags |= flag;
 303                                 else
 304                                         testcase->flags &= ~flag;
 305                                 ++found;
 306                         }
 307                 }
 308         }
 309         return found;
 310 }
 311
 312 static void
 313 usage(struct testgroup_t *groups, int list_groups)
 314 {
 315         puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
 316         puts("  Specify tests by name, or using a prefix ending with '..'");
 317         puts("  To skip a test, prefix its name with a colon.");
 318         puts("  To enable a disabled test, prefix its name with a plus.");
 319         puts("  Use --list-tests for a list of tests.");
 320         if (list_groups) {
 321                 puts("Known tests are:");
 322                 tinytest_set_flag_(groups, "..", 1, 0);
 323         }
 324         exit(0);
 325 }
 326
 327 static int
 328 process_test_alias(struct testgroup_t *groups, const char *test)
 329 {
 330         int i, j, n, r;
 331         for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
 332                 if (!strcmp(cfg_aliases[i].name, test)) {
 333                         n = 0;
 334                         for (j = 0; cfg_aliases[i].tests[j]; ++j) {
 335                                 r = process_test_option(groups, cfg_aliases[i].tests[j]);
 336                                 if (r<0)
 337                                         return -1;
 338                                 n += r;
 339                         }
 340                         return n;
 341                 }
 342         }
 343         printf("No such test alias as @%s!",test);
 344         return -1;
 345 }
 346
 347 static int
 348 process_test_option(struct testgroup_t *groups, const char *test)
 349 {
 350         int flag = TT_ENABLED_;
 351         int n = 0;
 352         if (test[0] == '@') {
 353                 return process_test_alias(groups, test + 1);
 354         } else if (test[0] == ':') {
 355                 ++test;
 356                 flag = TT_SKIP;
 357         } else if (test[0] == '+') {
 358                 ++test;
 359                 ++n;
 360                 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
 361                         printf("No such test as %s!\n", test);
 362                         return -1;
 363                 }
 364         } else {
 365                 ++n;
 366         }
 367         if (!tinytest_set_flag_(groups, test, 1, flag)) {
 368                 printf("No such test as %s!\n", test);
 369                 return -1;
 370         }
 371         return n;
 372 }
 373
 374 void
 375 tinytest_set_aliases(const struct testlist_alias_t *aliases)
 376 {
 377         cfg_aliases = aliases;
 378 }
 379
 380 int
 381 tinytest_main(int c, const char **v, struct testgroup_t *groups)
 382 {
 383         int i, j, n=0;
 384
 385 #ifdef _WIN32
 386         const char *sp = strrchr(v[0], '.');
 387         const char *extension = "";
 388         if (!sp || stricmp(sp, ".exe"))
 389                 extension = ".exe"; /* Add an exe so CreateProcess will work */
 390         snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
 391         commandname[MAX_PATH]='\0';
 392 #endif
 393         for (i=1; i<c; ++i) {
 394                 if (v[i][0] == '-') {
 395                         if (!strcmp(v[i], "--RUNNING-FORKED")) {
 396                                 opt_forked = 1;
 397                         } else if (!strcmp(v[i], "--no-fork")) {
 398                                 opt_nofork = 1;
 399                         } else if (!strcmp(v[i], "--quiet")) {
 400                                 opt_verbosity = -1;
 401                                 verbosity_flag = "--quiet";
 402                         } else if (!strcmp(v[i], "--verbose")) {
 403                                 opt_verbosity = 2;
 404                                 verbosity_flag = "--verbose";
 405                         } else if (!strcmp(v[i], "--terse")) {
 406                                 opt_verbosity = 0;
 407                                 verbosity_flag = "--terse";
 408                         } else if (!strcmp(v[i], "--help")) {
 409                                 usage(groups, 0);
 410                         } else if (!strcmp(v[i], "--list-tests")) {
 411                                 usage(groups, 1);
 412                         } else {
 413                                 printf("Unknown option %s.  Try --help\n",v[i]);
 414                                 return -1;
 415                         }
 416                 } else {
 417                         int r = process_test_option(groups, v[i]);
 418                         if (r<0)
 419                                 return -1;
 420                         n += r;
 421                 }
 422         }
 423         if (!n)
 424                 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
 425
 426 #ifdef _IONBF
 427         setvbuf(stdout, NULL, _IONBF, 0);
 428 #endif
 429
 430         ++in_tinytest_main;
 431         for (i=0; groups[i].prefix; ++i)
 432                 for (j=0; groups[i].cases[j].name; ++j)
 433                         if (groups[i].cases[j].flags & TT_ENABLED_)
 434                                 testcase_run_one(&groups[i],
 435                                                  &groups[i].cases[j]);
 436
 437         --in_tinytest_main;
 438
 439         if (opt_verbosity==0)
 440                 puts("");
 441
 442         if (n_bad)
 443                 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
 444                        n_bad+n_ok,n_skipped);
 445         else if (opt_verbosity >= 1)
 446                 printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
 447
 448         return (n_bad == 0) ? 0 : 1;
 449 }
 450
 451 int
 452 tinytest_get_verbosity_(void)
 453 {
 454         return opt_verbosity;
 455 }
 456
 457 void
 458 tinytest_set_test_failed_(void)
 459 {
 460         if (opt_verbosity <= 0 && cur_test_name) {
 461                 if (opt_verbosity==0) puts("");
 462                 printf("%s%s: ", cur_test_prefix, cur_test_name);
 463                 cur_test_name = NULL;
 464         }
 465         cur_test_outcome = 0;
 466 }
 467
 468 void
 469 tinytest_set_test_skipped_(void)
 470 {
 471         if (cur_test_outcome==OK)
 472                 cur_test_outcome = SKIP;
 473 }
 474
 475 char *
 476 tinytest_format_hex_(const void *val_, unsigned long len)
 477 {
 478         const unsigned char *val = val_;
 479         char *result, *cp;
 480         size_t i;
 481
 482         if (!val)
 483                 return strdup("null");
 484         if (!(result = malloc(len*2+1)))
 485                 return strdup("<allocation failure>");
 486         cp = result;
 487         for (i=0;i<len;++i) {
 488                 *cp++ = "0123456789ABCDEF"[val[i] >> 4];
 489                 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
 490         }
 491         *cp = 0;
 492         return result;
 493 }