contrib/jemalloc/src/malloc_io.c

   1 #define JEMALLOC_MALLOC_IO_C_
   2 #include "jemalloc/internal/jemalloc_preamble.h"
   3 #include "jemalloc/internal/jemalloc_internal_includes.h"
   4
   5 #include "jemalloc/internal/malloc_io.h"
   6 #include "jemalloc/internal/util.h"
   7
   8 #ifdef assert
   9 #  undef assert
  10 #endif
  11 #ifdef not_reached
  12 #  undef not_reached
  13 #endif
  14 #ifdef not_implemented
  15 #  undef not_implemented
  16 #endif
  17 #ifdef assert_not_implemented
  18 #  undef assert_not_implemented
  19 #endif
  20
  21 /*
  22  * Define simple versions of assertion macros that won't recurse in case
  23  * of assertion failures in malloc_*printf().
  24  */
  25 #define assert(e) do {                                                  \
  26         if (config_debug && !(e)) {                                     \
  27                 malloc_write("<jemalloc>: Failed assertion\n");         \
  28                 abort();                                                \
  29         }                                                               \
  30 } while (0)
  31
  32 #define not_reached() do {                                              \
  33         if (config_debug) {                                             \
  34                 malloc_write("<jemalloc>: Unreachable code reached\n"); \
  35                 abort();                                                \
  36         }                                                               \
  37         unreachable();                                                  \
  38 } while (0)
  39
  40 #define not_implemented() do {                                          \
  41         if (config_debug) {                                             \
  42                 malloc_write("<jemalloc>: Not implemented\n");          \
  43                 abort();                                                \
  44         }                                                               \
  45 } while (0)
  46
  47 #define assert_not_implemented(e) do {                                  \
  48         if (unlikely(config_debug && !(e))) {                           \
  49                 not_implemented();                                      \
  50         }                                                               \
  51 } while (0)
  52
  53 /******************************************************************************/
  54 /* Function prototypes for non-inline static functions. */
  55
  56 static void wrtmessage(void *cbopaque, const char *s);
  57 #define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1)
  58 static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s,
  59     size_t *slen_p);
  60 #define D2S_BUFSIZE (1 + U2S_BUFSIZE)
  61 static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p);
  62 #define O2S_BUFSIZE (1 + U2S_BUFSIZE)
  63 static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p);
  64 #define X2S_BUFSIZE (2 + U2S_BUFSIZE)
  65 static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
  66     size_t *slen_p);
  67
  68 /******************************************************************************/
  69
  70 /* malloc_message() setup. */
  71 static void
  72 wrtmessage(void *cbopaque, const char *s) {
  73 #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write)
  74         /*
  75          * Use syscall(2) rather than write(2) when possible in order to avoid
  76          * the possibility of memory allocation within libc.  This is necessary
  77          * on FreeBSD; most operating systems do not have this problem though.
  78          *
  79          * syscall() returns long or int, depending on platform, so capture the
  80          * unused result in the widest plausible type to avoid compiler
  81          * warnings.
  82          */
  83         UNUSED long result = syscall(SYS_write, STDERR_FILENO, s, strlen(s));
  84 #else
  85         UNUSED ssize_t result = write(STDERR_FILENO, s, strlen(s));
  86 #endif
  87 }
  88
  89 JEMALLOC_EXPORT void    (*je_malloc_message)(void *, const char *s);
  90
  91 JEMALLOC_ATTR(visibility("hidden"))
  92 void
  93 wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) {
  94
  95         wrtmessage(NULL, s1);
  96         wrtmessage(NULL, s2);
  97         wrtmessage(NULL, s3);
  98         wrtmessage(NULL, s4);
  99 }
 100
 101 void    (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
 102     const char *s4) = wrtmessage_1_0;
 103 __sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
 104
 105 /*
 106  * Wrapper around malloc_message() that avoids the need for
 107  * je_malloc_message(...) throughout the code.
 108  */
 109 void
 110 malloc_write(const char *s) {
 111         if (je_malloc_message != NULL) {
 112                 je_malloc_message(NULL, s);
 113         } else {
 114                 wrtmessage(NULL, s);
 115         }
 116 }
 117
 118 /*
 119  * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
 120  * provide a wrapper.
 121  */
 122 int
 123 buferror(int err, char *buf, size_t buflen) {
 124 #ifdef _WIN32
 125         FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0,
 126             (LPSTR)buf, (DWORD)buflen, NULL);
 127         return 0;
 128 #elif defined(__GLIBC__) && defined(_GNU_SOURCE)
 129         char *b = strerror_r(err, buf, buflen);
 130         if (b != buf) {
 131                 strncpy(buf, b, buflen);
 132                 buf[buflen-1] = '\0';
 133         }
 134         return 0;
 135 #else
 136         return strerror_r(err, buf, buflen);
 137 #endif
 138 }
 139
 140 uintmax_t
 141 malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) {
 142         uintmax_t ret, digit;
 143         unsigned b;
 144         bool neg;
 145         const char *p, *ns;
 146
 147         p = nptr;
 148         if (base < 0 || base == 1 || base > 36) {
 149                 ns = p;
 150                 set_errno(EINVAL);
 151                 ret = UINTMAX_MAX;
 152                 goto label_return;
 153         }
 154         b = base;
 155
 156         /* Swallow leading whitespace and get sign, if any. */
 157         neg = false;
 158         while (true) {
 159                 switch (*p) {
 160                 case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
 161                         p++;
 162                         break;
 163                 case '-':
 164                         neg = true;
 165                         /* Fall through. */
 166                 case '+':
 167                         p++;
 168                         /* Fall through. */
 169                 default:
 170                         goto label_prefix;
 171                 }
 172         }
 173
 174         /* Get prefix, if any. */
 175         label_prefix:
 176         /*
 177          * Note where the first non-whitespace/sign character is so that it is
 178          * possible to tell whether any digits are consumed (e.g., "  0" vs.
 179          * "  -x").
 180          */
 181         ns = p;
 182         if (*p == '0') {
 183                 switch (p[1]) {
 184                 case '0': case '1': case '2': case '3': case '4': case '5':
 185                 case '6': case '7':
 186                         if (b == 0) {
 187                                 b = 8;
 188                         }
 189                         if (b == 8) {
 190                                 p++;
 191                         }
 192                         break;
 193                 case 'X': case 'x':
 194                         switch (p[2]) {
 195                         case '0': case '1': case '2': case '3': case '4':
 196                         case '5': case '6': case '7': case '8': case '9':
 197                         case 'A': case 'B': case 'C': case 'D': case 'E':
 198                         case 'F':
 199                         case 'a': case 'b': case 'c': case 'd': case 'e':
 200                         case 'f':
 201                                 if (b == 0) {
 202                                         b = 16;
 203                                 }
 204                                 if (b == 16) {
 205                                         p += 2;
 206                                 }
 207                                 break;
 208                         default:
 209                                 break;
 210                         }
 211                         break;
 212                 default:
 213                         p++;
 214                         ret = 0;
 215                         goto label_return;
 216                 }
 217         }
 218         if (b == 0) {
 219                 b = 10;
 220         }
 221
 222         /* Convert. */
 223         ret = 0;
 224         while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b)
 225             || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b)
 226             || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) {
 227                 uintmax_t pret = ret;
 228                 ret *= b;
 229                 ret += digit;
 230                 if (ret < pret) {
 231                         /* Overflow. */
 232                         set_errno(ERANGE);
 233                         ret = UINTMAX_MAX;
 234                         goto label_return;
 235                 }
 236                 p++;
 237         }
 238         if (neg) {
 239                 ret = (uintmax_t)(-((intmax_t)ret));
 240         }
 241
 242         if (p == ns) {
 243                 /* No conversion performed. */
 244                 set_errno(EINVAL);
 245                 ret = UINTMAX_MAX;
 246                 goto label_return;
 247         }
 248
 249 label_return:
 250         if (endptr != NULL) {
 251                 if (p == ns) {
 252                         /* No characters were converted. */
 253                         *endptr = (char *)nptr;
 254                 } else {
 255                         *endptr = (char *)p;
 256                 }
 257         }
 258         return ret;
 259 }
 260
 261 static char *
 262 u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) {
 263         unsigned i;
 264
 265         i = U2S_BUFSIZE - 1;
 266         s[i] = '\0';
 267         switch (base) {
 268         case 10:
 269                 do {
 270                         i--;
 271                         s[i] = "0123456789"[x % (uint64_t)10];
 272                         x /= (uint64_t)10;
 273                 } while (x > 0);
 274                 break;
 275         case 16: {
 276                 const char *digits = (uppercase)
 277                     ? "0123456789ABCDEF"
 278                     : "0123456789abcdef";
 279
 280                 do {
 281                         i--;
 282                         s[i] = digits[x & 0xf];
 283                         x >>= 4;
 284                 } while (x > 0);
 285                 break;
 286         } default: {
 287                 const char *digits = (uppercase)
 288                     ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 289                     : "0123456789abcdefghijklmnopqrstuvwxyz";
 290
 291                 assert(base >= 2 && base <= 36);
 292                 do {
 293                         i--;
 294                         s[i] = digits[x % (uint64_t)base];
 295                         x /= (uint64_t)base;
 296                 } while (x > 0);
 297         }}
 298
 299         *slen_p = U2S_BUFSIZE - 1 - i;
 300         return &s[i];
 301 }
 302
 303 static char *
 304 d2s(intmax_t x, char sign, char *s, size_t *slen_p) {
 305         bool neg;
 306
 307         if ((neg = (x < 0))) {
 308                 x = -x;
 309         }
 310         s = u2s(x, 10, false, s, slen_p);
 311         if (neg) {
 312                 sign = '-';
 313         }
 314         switch (sign) {
 315         case '-':
 316                 if (!neg) {
 317                         break;
 318                 }
 319                 /* Fall through. */
 320         case ' ':
 321         case '+':
 322                 s--;
 323                 (*slen_p)++;
 324                 *s = sign;
 325                 break;
 326         default: not_reached();
 327         }
 328         return s;
 329 }
 330
 331 static char *
 332 o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) {
 333         s = u2s(x, 8, false, s, slen_p);
 334         if (alt_form && *s != '0') {
 335                 s--;
 336                 (*slen_p)++;
 337                 *s = '0';
 338         }
 339         return s;
 340 }
 341
 342 static char *
 343 x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) {
 344         s = u2s(x, 16, uppercase, s, slen_p);
 345         if (alt_form) {
 346                 s -= 2;
 347                 (*slen_p) += 2;
 348                 memcpy(s, uppercase ? "0X" : "0x", 2);
 349         }
 350         return s;
 351 }
 352
 353 size_t
 354 malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
 355         size_t i;
 356         const char *f;
 357
 358 #define APPEND_C(c) do {                                                \
 359         if (i < size) {                                                 \
 360                 str[i] = (c);                                           \
 361         }                                                               \
 362         i++;                                                            \
 363 } while (0)
 364 #define APPEND_S(s, slen) do {                                          \
 365         if (i < size) {                                                 \
 366                 size_t cpylen = (slen <= size - i) ? slen : size - i;   \
 367                 memcpy(&str[i], s, cpylen);                             \
 368         }                                                               \
 369         i += slen;                                                      \
 370 } while (0)
 371 #define APPEND_PADDED_S(s, slen, width, left_justify) do {              \
 372         /* Left padding. */                                             \
 373         size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ?  \
 374             (size_t)width - slen : 0);                                  \
 375         if (!left_justify && pad_len != 0) {                            \
 376                 size_t j;                                               \
 377                 for (j = 0; j < pad_len; j++) {                         \
 378                         APPEND_C(' ');                                  \
 379                 }                                                       \
 380         }                                                               \
 381         /* Value. */                                                    \
 382         APPEND_S(s, slen);                                              \
 383         /* Right padding. */                                            \
 384         if (left_justify && pad_len != 0) {                             \
 385                 size_t j;                                               \
 386                 for (j = 0; j < pad_len; j++) {                         \
 387                         APPEND_C(' ');                                  \
 388                 }                                                       \
 389         }                                                               \
 390 } while (0)
 391 #define GET_ARG_NUMERIC(val, len) do {                                  \
 392         switch (len) {                                                  \
 393         case '?':                                                       \
 394                 val = va_arg(ap, int);                                  \
 395                 break;                                                  \
 396         case '?' | 0x80:                                                \
 397                 val = va_arg(ap, unsigned int);                         \
 398                 break;                                                  \
 399         case 'l':                                                       \
 400                 val = va_arg(ap, long);                                 \
 401                 break;                                                  \
 402         case 'l' | 0x80:                                                \
 403                 val = va_arg(ap, unsigned long);                        \
 404                 break;                                                  \
 405         case 'q':                                                       \
 406                 val = va_arg(ap, long long);                            \
 407                 break;                                                  \
 408         case 'q' | 0x80:                                                \
 409                 val = va_arg(ap, unsigned long long);                   \
 410                 break;                                                  \
 411         case 'j':                                                       \
 412                 val = va_arg(ap, intmax_t);                             \
 413                 break;                                                  \
 414         case 'j' | 0x80:                                                \
 415                 val = va_arg(ap, uintmax_t);                            \
 416                 break;                                                  \
 417         case 't':                                                       \
 418                 val = va_arg(ap, ptrdiff_t);                            \
 419                 break;                                                  \
 420         case 'z':                                                       \
 421                 val = va_arg(ap, ssize_t);                              \
 422                 break;                                                  \
 423         case 'z' | 0x80:                                                \
 424                 val = va_arg(ap, size_t);                               \
 425                 break;                                                  \
 426         case 'p': /* Synthetic; used for %p. */                         \
 427                 val = va_arg(ap, uintptr_t);                            \
 428                 break;                                                  \
 429         default:                                                        \
 430                 not_reached();                                          \
 431                 val = 0;                                                \
 432         }                                                               \
 433 } while (0)
 434
 435         i = 0;
 436         f = format;
 437         while (true) {
 438                 switch (*f) {
 439                 case '\0': goto label_out;
 440                 case '%': {
 441                         bool alt_form = false;
 442                         bool left_justify = false;
 443                         bool plus_space = false;
 444                         bool plus_plus = false;
 445                         int prec = -1;
 446                         int width = -1;
 447                         unsigned char len = '?';
 448                         char *s;
 449                         size_t slen;
 450
 451                         f++;
 452                         /* Flags. */
 453                         while (true) {
 454                                 switch (*f) {
 455                                 case '#':
 456                                         assert(!alt_form);
 457                                         alt_form = true;
 458                                         break;
 459                                 case '-':
 460                                         assert(!left_justify);
 461                                         left_justify = true;
 462                                         break;
 463                                 case ' ':
 464                                         assert(!plus_space);
 465                                         plus_space = true;
 466                                         break;
 467                                 case '+':
 468                                         assert(!plus_plus);
 469                                         plus_plus = true;
 470                                         break;
 471                                 default: goto label_width;
 472                                 }
 473                                 f++;
 474                         }
 475                         /* Width. */
 476                         label_width:
 477                         switch (*f) {
 478                         case '*':
 479                                 width = va_arg(ap, int);
 480                                 f++;
 481                                 if (width < 0) {
 482                                         left_justify = true;
 483                                         width = -width;
 484                                 }
 485                                 break;
 486                         case '0': case '1': case '2': case '3': case '4':
 487                         case '5': case '6': case '7': case '8': case '9': {
 488                                 uintmax_t uwidth;
 489                                 set_errno(0);
 490                                 uwidth = malloc_strtoumax(f, (char **)&f, 10);
 491                                 assert(uwidth != UINTMAX_MAX || get_errno() !=
 492                                     ERANGE);
 493                                 width = (int)uwidth;
 494                                 break;
 495                         } default:
 496                                 break;
 497                         }
 498                         /* Width/precision separator. */
 499                         if (*f == '.') {
 500                                 f++;
 501                         } else {
 502                                 goto label_length;
 503                         }
 504                         /* Precision. */
 505                         switch (*f) {
 506                         case '*':
 507                                 prec = va_arg(ap, int);
 508                                 f++;
 509                                 break;
 510                         case '0': case '1': case '2': case '3': case '4':
 511                         case '5': case '6': case '7': case '8': case '9': {
 512                                 uintmax_t uprec;
 513                                 set_errno(0);
 514                                 uprec = malloc_strtoumax(f, (char **)&f, 10);
 515                                 assert(uprec != UINTMAX_MAX || get_errno() !=
 516                                     ERANGE);
 517                                 prec = (int)uprec;
 518                                 break;
 519                         }
 520                         default: break;
 521                         }
 522                         /* Length. */
 523                         label_length:
 524                         switch (*f) {
 525                         case 'l':
 526                                 f++;
 527                                 if (*f == 'l') {
 528                                         len = 'q';
 529                                         f++;
 530                                 } else {
 531                                         len = 'l';
 532                                 }
 533                                 break;
 534                         case 'q': case 'j': case 't': case 'z':
 535                                 len = *f;
 536                                 f++;
 537                                 break;
 538                         default: break;
 539                         }
 540                         /* Conversion specifier. */
 541                         switch (*f) {
 542                         case '%':
 543                                 /* %% */
 544                                 APPEND_C(*f);
 545                                 f++;
 546                                 break;
 547                         case 'd': case 'i': {
 548                                 intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 549                                 char buf[D2S_BUFSIZE];
 550
 551                                 GET_ARG_NUMERIC(val, len);
 552                                 s = d2s(val, (plus_plus ? '+' : (plus_space ?
 553                                     ' ' : '-')), buf, &slen);
 554                                 APPEND_PADDED_S(s, slen, width, left_justify);
 555                                 f++;
 556                                 break;
 557                         } case 'o': {
 558                                 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 559                                 char buf[O2S_BUFSIZE];
 560
 561                                 GET_ARG_NUMERIC(val, len | 0x80);
 562                                 s = o2s(val, alt_form, buf, &slen);
 563                                 APPEND_PADDED_S(s, slen, width, left_justify);
 564                                 f++;
 565                                 break;
 566                         } case 'u': {
 567                                 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 568                                 char buf[U2S_BUFSIZE];
 569
 570                                 GET_ARG_NUMERIC(val, len | 0x80);
 571                                 s = u2s(val, 10, false, buf, &slen);
 572                                 APPEND_PADDED_S(s, slen, width, left_justify);
 573                                 f++;
 574                                 break;
 575                         } case 'x': case 'X': {
 576                                 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 577                                 char buf[X2S_BUFSIZE];
 578
 579                                 GET_ARG_NUMERIC(val, len | 0x80);
 580                                 s = x2s(val, alt_form, *f == 'X', buf, &slen);
 581                                 APPEND_PADDED_S(s, slen, width, left_justify);
 582                                 f++;
 583                                 break;
 584                         } case 'c': {
 585                                 unsigned char val;
 586                                 char buf[2];
 587
 588                                 assert(len == '?' || len == 'l');
 589                                 assert_not_implemented(len != 'l');
 590                                 val = va_arg(ap, int);
 591                                 buf[0] = val;
 592                                 buf[1] = '\0';
 593                                 APPEND_PADDED_S(buf, 1, width, left_justify);
 594                                 f++;
 595                                 break;
 596                         } case 's':
 597                                 assert(len == '?' || len == 'l');
 598                                 assert_not_implemented(len != 'l');
 599                                 s = va_arg(ap, char *);
 600                                 slen = (prec < 0) ? strlen(s) : (size_t)prec;
 601                                 APPEND_PADDED_S(s, slen, width, left_justify);
 602                                 f++;
 603                                 break;
 604                         case 'p': {
 605                                 uintmax_t val;
 606                                 char buf[X2S_BUFSIZE];
 607
 608                                 GET_ARG_NUMERIC(val, 'p');
 609                                 s = x2s(val, true, false, buf, &slen);
 610                                 APPEND_PADDED_S(s, slen, width, left_justify);
 611                                 f++;
 612                                 break;
 613                         } default: not_reached();
 614                         }
 615                         break;
 616                 } default: {
 617                         APPEND_C(*f);
 618                         f++;
 619                         break;
 620                 }}
 621         }
 622         label_out:
 623         if (i < size) {
 624                 str[i] = '\0';
 625         } else {
 626                 str[size - 1] = '\0';
 627         }
 628
 629 #undef APPEND_C
 630 #undef APPEND_S
 631 #undef APPEND_PADDED_S
 632 #undef GET_ARG_NUMERIC
 633         return i;
 634 }
 635
 636 JEMALLOC_FORMAT_PRINTF(3, 4)
 637 size_t
 638 malloc_snprintf(char *str, size_t size, const char *format, ...) {
 639         size_t ret;
 640         va_list ap;
 641
 642         va_start(ap, format);
 643         ret = malloc_vsnprintf(str, size, format, ap);
 644         va_end(ap);
 645
 646         return ret;
 647 }
 648
 649 void
 650 malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
 651     const char *format, va_list ap) {
 652         char buf[MALLOC_PRINTF_BUFSIZE];
 653
 654         if (write_cb == NULL) {
 655                 /*
 656                  * The caller did not provide an alternate write_cb callback
 657                  * function, so use the default one.  malloc_write() is an
 658                  * inline function, so use malloc_message() directly here.
 659                  */
 660                 write_cb = (je_malloc_message != NULL) ? je_malloc_message :
 661                     wrtmessage;
 662                 cbopaque = NULL;
 663         }
 664
 665         malloc_vsnprintf(buf, sizeof(buf), format, ap);
 666         write_cb(cbopaque, buf);
 667 }
 668
 669 /*
 670  * Print to a callback function in such a way as to (hopefully) avoid memory
 671  * allocation.
 672  */
 673 JEMALLOC_FORMAT_PRINTF(3, 4)
 674 void
 675 malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
 676     const char *format, ...) {
 677         va_list ap;
 678
 679         va_start(ap, format);
 680         malloc_vcprintf(write_cb, cbopaque, format, ap);
 681         va_end(ap);
 682 }
 683
 684 /* Print to stderr in such a way as to avoid memory allocation. */
 685 JEMALLOC_FORMAT_PRINTF(1, 2)
 686 void
 687 malloc_printf(const char *format, ...) {
 688         va_list ap;
 689
 690         va_start(ap, format);
 691         malloc_vcprintf(NULL, NULL, format, ap);
 692         va_end(ap);
 693 }
 694
 695 /*
 696  * Restore normal assertion macros, in order to make it possible to compile all
 697  * C files as a single concatenation.
 698  */
 699 #undef assert
 700 #undef not_reached
 701 #undef not_implemented
 702 #undef assert_not_implemented
 703 #include "jemalloc/internal/assert.h"