contrib/jemalloc/src/malloc_io.c

   1 #define JEMALLOC_MALLOC_IO_C_
   2 #include "jemalloc/internal/jemalloc_preamble.h"
   3 #include "jemalloc/internal/jemalloc_internal_includes.h"
   4
   5 #include "jemalloc/internal/malloc_io.h"
   6 #include "jemalloc/internal/util.h"
   7
   8 #ifdef assert
   9 #  undef assert
  10 #endif
  11 #ifdef not_reached
  12 #  undef not_reached
  13 #endif
  14 #ifdef not_implemented
  15 #  undef not_implemented
  16 #endif
  17 #ifdef assert_not_implemented
  18 #  undef assert_not_implemented
  19 #endif
  20
  21 /*
  22  * Define simple versions of assertion macros that won't recurse in case
  23  * of assertion failures in malloc_*printf().
  24  */
  25 #define assert(e) do {                                                  \
  26         if (config_debug && !(e)) {                                     \
  27                 malloc_write("<jemalloc>: Failed assertion\n");         \
  28                 abort();                                                \
  29         }                                                               \
  30 } while (0)
  31
  32 #define not_reached() do {                                              \
  33         if (config_debug) {                                             \
  34                 malloc_write("<jemalloc>: Unreachable code reached\n"); \
  35                 abort();                                                \
  36         }                                                               \
  37         unreachable();                                                  \
  38 } while (0)
  39
  40 #define not_implemented() do {                                          \
  41         if (config_debug) {                                             \
  42                 malloc_write("<jemalloc>: Not implemented\n");          \
  43                 abort();                                                \
  44         }                                                               \
  45 } while (0)
  46
  47 #define assert_not_implemented(e) do {                                  \
  48         if (unlikely(config_debug && !(e))) {                           \
  49                 not_implemented();                                      \
  50         }                                                               \
  51 } while (0)
  52
  53 /******************************************************************************/
  54 /* Function prototypes for non-inline static functions. */
  55
  56 static void wrtmessage(void *cbopaque, const char *s);
  57 #define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1)
  58 static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s,
  59     size_t *slen_p);
  60 #define D2S_BUFSIZE (1 + U2S_BUFSIZE)
  61 static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p);
  62 #define O2S_BUFSIZE (1 + U2S_BUFSIZE)
  63 static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p);
  64 #define X2S_BUFSIZE (2 + U2S_BUFSIZE)
  65 static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
  66     size_t *slen_p);
  67
  68 /******************************************************************************/
  69
  70 /* malloc_message() setup. */
  71 static void
  72 wrtmessage(void *cbopaque, const char *s) {
  73         malloc_write_fd(STDERR_FILENO, s, strlen(s));
  74 }
  75
  76 JEMALLOC_EXPORT void    (*je_malloc_message)(void *, const char *s);
  77
  78 JEMALLOC_ATTR(visibility("hidden"))
  79 void
  80 wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) {
  81
  82         wrtmessage(NULL, s1);
  83         wrtmessage(NULL, s2);
  84         wrtmessage(NULL, s3);
  85         wrtmessage(NULL, s4);
  86 }
  87
  88 void    (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
  89     const char *s4) = wrtmessage_1_0;
  90 __sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
  91
  92 /*
  93  * Wrapper around malloc_message() that avoids the need for
  94  * je_malloc_message(...) throughout the code.
  95  */
  96 void
  97 malloc_write(const char *s) {
  98         if (je_malloc_message != NULL) {
  99                 je_malloc_message(NULL, s);
 100         } else {
 101                 wrtmessage(NULL, s);
 102         }
 103 }
 104
 105 /*
 106  * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
 107  * provide a wrapper.
 108  */
 109 int
 110 buferror(int err, char *buf, size_t buflen) {
 111 #ifdef _WIN32
 112         FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0,
 113             (LPSTR)buf, (DWORD)buflen, NULL);
 114         return 0;
 115 #elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE)
 116         char *b = strerror_r(err, buf, buflen);
 117         if (b != buf) {
 118                 strncpy(buf, b, buflen);
 119                 buf[buflen-1] = '\0';
 120         }
 121         return 0;
 122 #else
 123         return strerror_r(err, buf, buflen);
 124 #endif
 125 }
 126
 127 uintmax_t
 128 malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) {
 129         uintmax_t ret, digit;
 130         unsigned b;
 131         bool neg;
 132         const char *p, *ns;
 133
 134         p = nptr;
 135         if (base < 0 || base == 1 || base > 36) {
 136                 ns = p;
 137                 set_errno(EINVAL);
 138                 ret = UINTMAX_MAX;
 139                 goto label_return;
 140         }
 141         b = base;
 142
 143         /* Swallow leading whitespace and get sign, if any. */
 144         neg = false;
 145         while (true) {
 146                 switch (*p) {
 147                 case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
 148                         p++;
 149                         break;
 150                 case '-':
 151                         neg = true;
 152                         /* Fall through. */
 153                 case '+':
 154                         p++;
 155                         /* Fall through. */
 156                 default:
 157                         goto label_prefix;
 158                 }
 159         }
 160
 161         /* Get prefix, if any. */
 162         label_prefix:
 163         /*
 164          * Note where the first non-whitespace/sign character is so that it is
 165          * possible to tell whether any digits are consumed (e.g., "  0" vs.
 166          * "  -x").
 167          */
 168         ns = p;
 169         if (*p == '0') {
 170                 switch (p[1]) {
 171                 case '0': case '1': case '2': case '3': case '4': case '5':
 172                 case '6': case '7':
 173                         if (b == 0) {
 174                                 b = 8;
 175                         }
 176                         if (b == 8) {
 177                                 p++;
 178                         }
 179                         break;
 180                 case 'X': case 'x':
 181                         switch (p[2]) {
 182                         case '0': case '1': case '2': case '3': case '4':
 183                         case '5': case '6': case '7': case '8': case '9':
 184                         case 'A': case 'B': case 'C': case 'D': case 'E':
 185                         case 'F':
 186                         case 'a': case 'b': case 'c': case 'd': case 'e':
 187                         case 'f':
 188                                 if (b == 0) {
 189                                         b = 16;
 190                                 }
 191                                 if (b == 16) {
 192                                         p += 2;
 193                                 }
 194                                 break;
 195                         default:
 196                                 break;
 197                         }
 198                         break;
 199                 default:
 200                         p++;
 201                         ret = 0;
 202                         goto label_return;
 203                 }
 204         }
 205         if (b == 0) {
 206                 b = 10;
 207         }
 208
 209         /* Convert. */
 210         ret = 0;
 211         while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b)
 212             || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b)
 213             || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) {
 214                 uintmax_t pret = ret;
 215                 ret *= b;
 216                 ret += digit;
 217                 if (ret < pret) {
 218                         /* Overflow. */
 219                         set_errno(ERANGE);
 220                         ret = UINTMAX_MAX;
 221                         goto label_return;
 222                 }
 223                 p++;
 224         }
 225         if (neg) {
 226                 ret = (uintmax_t)(-((intmax_t)ret));
 227         }
 228
 229         if (p == ns) {
 230                 /* No conversion performed. */
 231                 set_errno(EINVAL);
 232                 ret = UINTMAX_MAX;
 233                 goto label_return;
 234         }
 235
 236 label_return:
 237         if (endptr != NULL) {
 238                 if (p == ns) {
 239                         /* No characters were converted. */
 240                         *endptr = (char *)nptr;
 241                 } else {
 242                         *endptr = (char *)p;
 243                 }
 244         }
 245         return ret;
 246 }
 247
 248 static char *
 249 u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) {
 250         unsigned i;
 251
 252         i = U2S_BUFSIZE - 1;
 253         s[i] = '\0';
 254         switch (base) {
 255         case 10:
 256                 do {
 257                         i--;
 258                         s[i] = "0123456789"[x % (uint64_t)10];
 259                         x /= (uint64_t)10;
 260                 } while (x > 0);
 261                 break;
 262         case 16: {
 263                 const char *digits = (uppercase)
 264                     ? "0123456789ABCDEF"
 265                     : "0123456789abcdef";
 266
 267                 do {
 268                         i--;
 269                         s[i] = digits[x & 0xf];
 270                         x >>= 4;
 271                 } while (x > 0);
 272                 break;
 273         } default: {
 274                 const char *digits = (uppercase)
 275                     ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 276                     : "0123456789abcdefghijklmnopqrstuvwxyz";
 277
 278                 assert(base >= 2 && base <= 36);
 279                 do {
 280                         i--;
 281                         s[i] = digits[x % (uint64_t)base];
 282                         x /= (uint64_t)base;
 283                 } while (x > 0);
 284         }}
 285
 286         *slen_p = U2S_BUFSIZE - 1 - i;
 287         return &s[i];
 288 }
 289
 290 static char *
 291 d2s(intmax_t x, char sign, char *s, size_t *slen_p) {
 292         bool neg;
 293
 294         if ((neg = (x < 0))) {
 295                 x = -x;
 296         }
 297         s = u2s(x, 10, false, s, slen_p);
 298         if (neg) {
 299                 sign = '-';
 300         }
 301         switch (sign) {
 302         case '-':
 303                 if (!neg) {
 304                         break;
 305                 }
 306                 /* Fall through. */
 307         case ' ':
 308         case '+':
 309                 s--;
 310                 (*slen_p)++;
 311                 *s = sign;
 312                 break;
 313         default: not_reached();
 314         }
 315         return s;
 316 }
 317
 318 static char *
 319 o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) {
 320         s = u2s(x, 8, false, s, slen_p);
 321         if (alt_form && *s != '0') {
 322                 s--;
 323                 (*slen_p)++;
 324                 *s = '0';
 325         }
 326         return s;
 327 }
 328
 329 static char *
 330 x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) {
 331         s = u2s(x, 16, uppercase, s, slen_p);
 332         if (alt_form) {
 333                 s -= 2;
 334                 (*slen_p) += 2;
 335                 memcpy(s, uppercase ? "0X" : "0x", 2);
 336         }
 337         return s;
 338 }
 339
 340 size_t
 341 malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
 342         size_t i;
 343         const char *f;
 344
 345 #define APPEND_C(c) do {                                                \
 346         if (i < size) {                                                 \
 347                 str[i] = (c);                                           \
 348         }                                                               \
 349         i++;                                                            \
 350 } while (0)
 351 #define APPEND_S(s, slen) do {                                          \
 352         if (i < size) {                                                 \
 353                 size_t cpylen = (slen <= size - i) ? slen : size - i;   \
 354                 memcpy(&str[i], s, cpylen);                             \
 355         }                                                               \
 356         i += slen;                                                      \
 357 } while (0)
 358 #define APPEND_PADDED_S(s, slen, width, left_justify) do {              \
 359         /* Left padding. */                                             \
 360         size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ?  \
 361             (size_t)width - slen : 0);                                  \
 362         if (!left_justify && pad_len != 0) {                            \
 363                 size_t j;                                               \
 364                 for (j = 0; j < pad_len; j++) {                         \
 365                         APPEND_C(' ');                                  \
 366                 }                                                       \
 367         }                                                               \
 368         /* Value. */                                                    \
 369         APPEND_S(s, slen);                                              \
 370         /* Right padding. */                                            \
 371         if (left_justify && pad_len != 0) {                             \
 372                 size_t j;                                               \
 373                 for (j = 0; j < pad_len; j++) {                         \
 374                         APPEND_C(' ');                                  \
 375                 }                                                       \
 376         }                                                               \
 377 } while (0)
 378 #define GET_ARG_NUMERIC(val, len) do {                                  \
 379         switch (len) {                                                  \
 380         case '?':                                                       \
 381                 val = va_arg(ap, int);                                  \
 382                 break;                                                  \
 383         case '?' | 0x80:                                                \
 384                 val = va_arg(ap, unsigned int);                         \
 385                 break;                                                  \
 386         case 'l':                                                       \
 387                 val = va_arg(ap, long);                                 \
 388                 break;                                                  \
 389         case 'l' | 0x80:                                                \
 390                 val = va_arg(ap, unsigned long);                        \
 391                 break;                                                  \
 392         case 'q':                                                       \
 393                 val = va_arg(ap, long long);                            \
 394                 break;                                                  \
 395         case 'q' | 0x80:                                                \
 396                 val = va_arg(ap, unsigned long long);                   \
 397                 break;                                                  \
 398         case 'j':                                                       \
 399                 val = va_arg(ap, intmax_t);                             \
 400                 break;                                                  \
 401         case 'j' | 0x80:                                                \
 402                 val = va_arg(ap, uintmax_t);                            \
 403                 break;                                                  \
 404         case 't':                                                       \
 405                 val = va_arg(ap, ptrdiff_t);                            \
 406                 break;                                                  \
 407         case 'z':                                                       \
 408                 val = va_arg(ap, ssize_t);                              \
 409                 break;                                                  \
 410         case 'z' | 0x80:                                                \
 411                 val = va_arg(ap, size_t);                               \
 412                 break;                                                  \
 413         case 'p': /* Synthetic; used for %p. */                         \
 414                 val = va_arg(ap, uintptr_t);                            \
 415                 break;                                                  \
 416         default:                                                        \
 417                 not_reached();                                          \
 418                 val = 0;                                                \
 419         }                                                               \
 420 } while (0)
 421
 422         i = 0;
 423         f = format;
 424         while (true) {
 425                 switch (*f) {
 426                 case '\0': goto label_out;
 427                 case '%': {
 428                         bool alt_form = false;
 429                         bool left_justify = false;
 430                         bool plus_space = false;
 431                         bool plus_plus = false;
 432                         int prec = -1;
 433                         int width = -1;
 434                         unsigned char len = '?';
 435                         char *s;
 436                         size_t slen;
 437
 438                         f++;
 439                         /* Flags. */
 440                         while (true) {
 441                                 switch (*f) {
 442                                 case '#':
 443                                         assert(!alt_form);
 444                                         alt_form = true;
 445                                         break;
 446                                 case '-':
 447                                         assert(!left_justify);
 448                                         left_justify = true;
 449                                         break;
 450                                 case ' ':
 451                                         assert(!plus_space);
 452                                         plus_space = true;
 453                                         break;
 454                                 case '+':
 455                                         assert(!plus_plus);
 456                                         plus_plus = true;
 457                                         break;
 458                                 default: goto label_width;
 459                                 }
 460                                 f++;
 461                         }
 462                         /* Width. */
 463                         label_width:
 464                         switch (*f) {
 465                         case '*':
 466                                 width = va_arg(ap, int);
 467                                 f++;
 468                                 if (width < 0) {
 469                                         left_justify = true;
 470                                         width = -width;
 471                                 }
 472                                 break;
 473                         case '0': case '1': case '2': case '3': case '4':
 474                         case '5': case '6': case '7': case '8': case '9': {
 475                                 uintmax_t uwidth;
 476                                 set_errno(0);
 477                                 uwidth = malloc_strtoumax(f, (char **)&f, 10);
 478                                 assert(uwidth != UINTMAX_MAX || get_errno() !=
 479                                     ERANGE);
 480                                 width = (int)uwidth;
 481                                 break;
 482                         } default:
 483                                 break;
 484                         }
 485                         /* Width/precision separator. */
 486                         if (*f == '.') {
 487                                 f++;
 488                         } else {
 489                                 goto label_length;
 490                         }
 491                         /* Precision. */
 492                         switch (*f) {
 493                         case '*':
 494                                 prec = va_arg(ap, int);
 495                                 f++;
 496                                 break;
 497                         case '0': case '1': case '2': case '3': case '4':
 498                         case '5': case '6': case '7': case '8': case '9': {
 499                                 uintmax_t uprec;
 500                                 set_errno(0);
 501                                 uprec = malloc_strtoumax(f, (char **)&f, 10);
 502                                 assert(uprec != UINTMAX_MAX || get_errno() !=
 503                                     ERANGE);
 504                                 prec = (int)uprec;
 505                                 break;
 506                         }
 507                         default: break;
 508                         }
 509                         /* Length. */
 510                         label_length:
 511                         switch (*f) {
 512                         case 'l':
 513                                 f++;
 514                                 if (*f == 'l') {
 515                                         len = 'q';
 516                                         f++;
 517                                 } else {
 518                                         len = 'l';
 519                                 }
 520                                 break;
 521                         case 'q': case 'j': case 't': case 'z':
 522                                 len = *f;
 523                                 f++;
 524                                 break;
 525                         default: break;
 526                         }
 527                         /* Conversion specifier. */
 528                         switch (*f) {
 529                         case '%':
 530                                 /* %% */
 531                                 APPEND_C(*f);
 532                                 f++;
 533                                 break;
 534                         case 'd': case 'i': {
 535                                 intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 536                                 char buf[D2S_BUFSIZE];
 537
 538                                 GET_ARG_NUMERIC(val, len);
 539                                 s = d2s(val, (plus_plus ? '+' : (plus_space ?
 540                                     ' ' : '-')), buf, &slen);
 541                                 APPEND_PADDED_S(s, slen, width, left_justify);
 542                                 f++;
 543                                 break;
 544                         } case 'o': {
 545                                 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 546                                 char buf[O2S_BUFSIZE];
 547
 548                                 GET_ARG_NUMERIC(val, len | 0x80);
 549                                 s = o2s(val, alt_form, buf, &slen);
 550                                 APPEND_PADDED_S(s, slen, width, left_justify);
 551                                 f++;
 552                                 break;
 553                         } case 'u': {
 554                                 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 555                                 char buf[U2S_BUFSIZE];
 556
 557                                 GET_ARG_NUMERIC(val, len | 0x80);
 558                                 s = u2s(val, 10, false, buf, &slen);
 559                                 APPEND_PADDED_S(s, slen, width, left_justify);
 560                                 f++;
 561                                 break;
 562                         } case 'x': case 'X': {
 563                                 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 564                                 char buf[X2S_BUFSIZE];
 565
 566                                 GET_ARG_NUMERIC(val, len | 0x80);
 567                                 s = x2s(val, alt_form, *f == 'X', buf, &slen);
 568                                 APPEND_PADDED_S(s, slen, width, left_justify);
 569                                 f++;
 570                                 break;
 571                         } case 'c': {
 572                                 unsigned char val;
 573                                 char buf[2];
 574
 575                                 assert(len == '?' || len == 'l');
 576                                 assert_not_implemented(len != 'l');
 577                                 val = va_arg(ap, int);
 578                                 buf[0] = val;
 579                                 buf[1] = '\0';
 580                                 APPEND_PADDED_S(buf, 1, width, left_justify);
 581                                 f++;
 582                                 break;
 583                         } case 's':
 584                                 assert(len == '?' || len == 'l');
 585                                 assert_not_implemented(len != 'l');
 586                                 s = va_arg(ap, char *);
 587                                 slen = (prec < 0) ? strlen(s) : (size_t)prec;
 588                                 APPEND_PADDED_S(s, slen, width, left_justify);
 589                                 f++;
 590                                 break;
 591                         case 'p': {
 592                                 uintmax_t val;
 593                                 char buf[X2S_BUFSIZE];
 594
 595                                 GET_ARG_NUMERIC(val, 'p');
 596                                 s = x2s(val, true, false, buf, &slen);
 597                                 APPEND_PADDED_S(s, slen, width, left_justify);
 598                                 f++;
 599                                 break;
 600                         } default: not_reached();
 601                         }
 602                         break;
 603                 } default: {
 604                         APPEND_C(*f);
 605                         f++;
 606                         break;
 607                 }}
 608         }
 609         label_out:
 610         if (i < size) {
 611                 str[i] = '\0';
 612         } else {
 613                 str[size - 1] = '\0';
 614         }
 615
 616 #undef APPEND_C
 617 #undef APPEND_S
 618 #undef APPEND_PADDED_S
 619 #undef GET_ARG_NUMERIC
 620         return i;
 621 }
 622
 623 JEMALLOC_FORMAT_PRINTF(3, 4)
 624 size_t
 625 malloc_snprintf(char *str, size_t size, const char *format, ...) {
 626         size_t ret;
 627         va_list ap;
 628
 629         va_start(ap, format);
 630         ret = malloc_vsnprintf(str, size, format, ap);
 631         va_end(ap);
 632
 633         return ret;
 634 }
 635
 636 void
 637 malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
 638     const char *format, va_list ap) {
 639         char buf[MALLOC_PRINTF_BUFSIZE];
 640
 641         if (write_cb == NULL) {
 642                 /*
 643                  * The caller did not provide an alternate write_cb callback
 644                  * function, so use the default one.  malloc_write() is an
 645                  * inline function, so use malloc_message() directly here.
 646                  */
 647                 write_cb = (je_malloc_message != NULL) ? je_malloc_message :
 648                     wrtmessage;
 649                 cbopaque = NULL;
 650         }
 651
 652         malloc_vsnprintf(buf, sizeof(buf), format, ap);
 653         write_cb(cbopaque, buf);
 654 }
 655
 656 /*
 657  * Print to a callback function in such a way as to (hopefully) avoid memory
 658  * allocation.
 659  */
 660 JEMALLOC_FORMAT_PRINTF(3, 4)
 661 void
 662 malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
 663     const char *format, ...) {
 664         va_list ap;
 665
 666         va_start(ap, format);
 667         malloc_vcprintf(write_cb, cbopaque, format, ap);
 668         va_end(ap);
 669 }
 670
 671 /* Print to stderr in such a way as to avoid memory allocation. */
 672 JEMALLOC_FORMAT_PRINTF(1, 2)
 673 void
 674 malloc_printf(const char *format, ...) {
 675         va_list ap;
 676
 677         va_start(ap, format);
 678         malloc_vcprintf(NULL, NULL, format, ap);
 679         va_end(ap);
 680 }
 681
 682 /*
 683  * Restore normal assertion macros, in order to make it possible to compile all
 684  * C files as a single concatenation.
 685  */
 686 #undef assert
 687 #undef not_reached
 688 #undef not_implemented
 689 #undef assert_not_implemented
 690 #include "jemalloc/internal/assert.h"