2 #include "jemalloc/internal/jemalloc_preamble.h"
3 #include "jemalloc/internal/jemalloc_internal_includes.h"
5 #include "jemalloc/internal/assert.h"
6 #include "jemalloc/internal/atomic.h"
7 #include "jemalloc/internal/ctl.h"
8 #include "jemalloc/internal/extent_dss.h"
9 #include "jemalloc/internal/extent_mmap.h"
10 #include "jemalloc/internal/hook.h"
11 #include "jemalloc/internal/jemalloc_internal_types.h"
12 #include "jemalloc/internal/log.h"
13 #include "jemalloc/internal/malloc_io.h"
14 #include "jemalloc/internal/mutex.h"
15 #include "jemalloc/internal/rtree.h"
16 #include "jemalloc/internal/safety_check.h"
17 #include "jemalloc/internal/sc.h"
18 #include "jemalloc/internal/spin.h"
19 #include "jemalloc/internal/sz.h"
20 #include "jemalloc/internal/ticker.h"
21 #include "jemalloc/internal/util.h"
23 /******************************************************************************/
26 /* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
27 const char *__malloc_options_1_0 = NULL;
28 __sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
30 /* Runtime configuration options. */
31 const char *je_malloc_conf
50 /* Intentionally default off, even with debug builds. */
51 bool opt_confirm_conf = false;
52 const char *opt_junk =
53 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
60 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
67 #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
74 bool opt_utrace = false;
75 bool opt_xmalloc = false;
76 bool opt_zero = false;
77 unsigned opt_narenas = 0;
81 /* Protects arenas initialization. */
82 malloc_mutex_t arenas_lock;
84 * Arenas that are used to service external requests. Not all elements of the
85 * arenas array are necessarily used; arenas are created lazily as needed.
87 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
88 * arenas. arenas[narenas_auto..narenas_total) are only used if the application
89 * takes some action to create them and allocate from them.
91 * Points to an arena_t.
93 JEMALLOC_ALIGNED(CACHELINE)
94 atomic_p_t arenas[MALLOCX_ARENA_LIMIT];
95 static atomic_u_t narenas_total; /* Use narenas_total_*(). */
96 /* Below three are read-only after initialization. */
97 static arena_t *a0; /* arenas[0]. */
98 unsigned narenas_auto;
99 unsigned manual_arena_base;
102 malloc_init_uninitialized = 3,
103 malloc_init_a0_initialized = 2,
104 malloc_init_recursible = 1,
105 malloc_init_initialized = 0 /* Common case --> jnz. */
107 static malloc_init_t malloc_init_state = malloc_init_uninitialized;
109 /* False should be the common case. Set to true to trigger initialization. */
110 bool malloc_slow = true;
112 /* When malloc_slow is true, set the corresponding bits for sanity check. */
114 flag_opt_junk_alloc = (1U),
115 flag_opt_junk_free = (1U << 1),
116 flag_opt_zero = (1U << 2),
117 flag_opt_utrace = (1U << 3),
118 flag_opt_xmalloc = (1U << 4)
120 static uint8_t malloc_slow_flags;
122 #ifdef JEMALLOC_THREADED_INIT
123 /* Used to let the initializing thread recursively allocate. */
124 # define NO_INITIALIZER ((unsigned long)0)
125 # define INITIALIZER pthread_self()
126 # define IS_INITIALIZER (malloc_initializer == pthread_self())
127 static pthread_t malloc_initializer = NO_INITIALIZER;
129 # define NO_INITIALIZER false
130 # define INITIALIZER true
131 # define IS_INITIALIZER malloc_initializer
132 static bool malloc_initializer = NO_INITIALIZER;
135 /* Used to avoid initialization races. */
137 #if _WIN32_WINNT >= 0x0600
138 static malloc_mutex_t init_lock = SRWLOCK_INIT;
140 static malloc_mutex_t init_lock;
141 static bool init_lock_initialized = false;
143 JEMALLOC_ATTR(constructor)
145 _init_init_lock(void) {
147 * If another constructor in the same binary is using mallctl to e.g.
148 * set up extent hooks, it may end up running before this one, and
149 * malloc_init_hard will crash trying to lock the uninitialized lock. So
150 * we force an initialization of the lock in malloc_init_hard as well.
151 * We don't try to care about atomicity of the accessed to the
152 * init_lock_initialized boolean, since it really only matters early in
153 * the process creation, before any separate thread normally starts
156 if (!init_lock_initialized) {
157 malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT,
158 malloc_mutex_rank_exclusive);
160 init_lock_initialized = true;
164 # pragma section(".CRT$XCU", read)
165 JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
166 static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
170 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
174 void *p; /* Input pointer (as in realloc(p, s)). */
175 size_t s; /* Request size. */
176 void *r; /* Result pointer. */
179 #ifdef JEMALLOC_UTRACE
180 # define UTRACE(a, b, c) do { \
181 if (unlikely(opt_utrace)) { \
182 int utrace_serrno = errno; \
183 malloc_utrace_t ut; \
187 utrace(&ut, sizeof(ut)); \
188 errno = utrace_serrno; \
192 # define UTRACE(a, b, c)
195 /* Whether encountered any invalid config options. */
196 static bool had_conf_error = false;
198 /******************************************************************************/
200 * Function prototypes for static functions that are referenced prior to
204 static bool malloc_init_hard_a0(void);
205 static bool malloc_init_hard(void);
207 /******************************************************************************/
209 * Begin miscellaneous support functions.
213 malloc_initialized(void) {
214 return (malloc_init_state == malloc_init_initialized);
217 JEMALLOC_ALWAYS_INLINE bool
218 malloc_init_a0(void) {
219 if (unlikely(malloc_init_state == malloc_init_uninitialized)) {
220 return malloc_init_hard_a0();
225 JEMALLOC_ALWAYS_INLINE bool
227 if (unlikely(!malloc_initialized()) && malloc_init_hard()) {
234 * The a0*() functions are used instead of i{d,}alloc() in situations that
235 * cannot tolerate TLS variable access.
239 a0ialloc(size_t size, bool zero, bool is_internal) {
240 if (unlikely(malloc_init_a0())) {
244 return iallocztm(TSDN_NULL, size, sz_size2index(size), zero, NULL,
245 is_internal, arena_get(TSDN_NULL, 0, true), true);
249 a0idalloc(void *ptr, bool is_internal) {
250 idalloctm(TSDN_NULL, ptr, NULL, NULL, is_internal, true);
254 a0malloc(size_t size) {
255 return a0ialloc(size, false, true);
259 a0dalloc(void *ptr) {
260 a0idalloc(ptr, true);
264 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive
265 * situations that cannot tolerate TLS variable access (TLS allocation and very
266 * early internal data structure initialization).
270 bootstrap_malloc(size_t size) {
271 if (unlikely(size == 0)) {
275 return a0ialloc(size, false, false);
279 bootstrap_calloc(size_t num, size_t size) {
282 num_size = num * size;
283 if (unlikely(num_size == 0)) {
284 assert(num == 0 || size == 0);
288 return a0ialloc(num_size, true, false);
292 bootstrap_free(void *ptr) {
293 if (unlikely(ptr == NULL)) {
297 a0idalloc(ptr, false);
301 arena_set(unsigned ind, arena_t *arena) {
302 atomic_store_p(&arenas[ind], arena, ATOMIC_RELEASE);
306 narenas_total_set(unsigned narenas) {
307 atomic_store_u(&narenas_total, narenas, ATOMIC_RELEASE);
311 narenas_total_inc(void) {
312 atomic_fetch_add_u(&narenas_total, 1, ATOMIC_RELEASE);
316 narenas_total_get(void) {
317 return atomic_load_u(&narenas_total, ATOMIC_ACQUIRE);
320 /* Create a new arena and insert it into the arenas array at index ind. */
322 arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
325 assert(ind <= narenas_total_get());
326 if (ind >= MALLOCX_ARENA_LIMIT) {
329 if (ind == narenas_total_get()) {
334 * Another thread may have already initialized arenas[ind] if it's an
337 arena = arena_get(tsdn, ind, false);
339 assert(arena_is_auto(arena));
343 /* Actually initialize the arena. */
344 arena = arena_new(tsdn, ind, extent_hooks);
350 arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) {
355 * Avoid creating a new background thread just for the huge arena, which
356 * purges eagerly by default.
358 if (have_background_thread && !arena_is_huge(ind)) {
359 if (background_thread_create(tsdn_tsd(tsdn), ind)) {
360 malloc_printf("<jemalloc>: error in background thread "
361 "creation for arena %u. Abort.\n", ind);
368 arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
371 malloc_mutex_lock(tsdn, &arenas_lock);
372 arena = arena_init_locked(tsdn, ind, extent_hooks);
373 malloc_mutex_unlock(tsdn, &arenas_lock);
375 arena_new_create_background_thread(tsdn, ind);
381 arena_bind(tsd_t *tsd, unsigned ind, bool internal) {
382 arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false);
383 arena_nthreads_inc(arena, internal);
386 tsd_iarena_set(tsd, arena);
388 tsd_arena_set(tsd, arena);
389 unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1,
391 tsd_binshards_t *bins = tsd_binshardsp_get(tsd);
392 for (unsigned i = 0; i < SC_NBINS; i++) {
393 assert(bin_infos[i].n_shards > 0 &&
394 bin_infos[i].n_shards <= BIN_SHARDS_MAX);
395 bins->binshard[i] = shard % bin_infos[i].n_shards;
401 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) {
402 arena_t *oldarena, *newarena;
404 oldarena = arena_get(tsd_tsdn(tsd), oldind, false);
405 newarena = arena_get(tsd_tsdn(tsd), newind, false);
406 arena_nthreads_dec(oldarena, false);
407 arena_nthreads_inc(newarena, false);
408 tsd_arena_set(tsd, newarena);
412 arena_unbind(tsd_t *tsd, unsigned ind, bool internal) {
415 arena = arena_get(tsd_tsdn(tsd), ind, false);
416 arena_nthreads_dec(arena, internal);
419 tsd_iarena_set(tsd, NULL);
421 tsd_arena_set(tsd, NULL);
426 arena_tdata_get_hard(tsd_t *tsd, unsigned ind) {
427 arena_tdata_t *tdata, *arenas_tdata_old;
428 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd);
429 unsigned narenas_tdata_old, i;
430 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd);
431 unsigned narenas_actual = narenas_total_get();
434 * Dissociate old tdata array (and set up for deallocation upon return)
437 if (arenas_tdata != NULL && narenas_tdata < narenas_actual) {
438 arenas_tdata_old = arenas_tdata;
439 narenas_tdata_old = narenas_tdata;
442 tsd_arenas_tdata_set(tsd, arenas_tdata);
443 tsd_narenas_tdata_set(tsd, narenas_tdata);
445 arenas_tdata_old = NULL;
446 narenas_tdata_old = 0;
449 /* Allocate tdata array if it's missing. */
450 if (arenas_tdata == NULL) {
451 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd);
452 narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1;
454 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) {
455 *arenas_tdata_bypassp = true;
456 arenas_tdata = (arena_tdata_t *)a0malloc(
457 sizeof(arena_tdata_t) * narenas_tdata);
458 *arenas_tdata_bypassp = false;
460 if (arenas_tdata == NULL) {
464 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp);
465 tsd_arenas_tdata_set(tsd, arenas_tdata);
466 tsd_narenas_tdata_set(tsd, narenas_tdata);
470 * Copy to tdata array. It's possible that the actual number of arenas
471 * has increased since narenas_total_get() was called above, but that
472 * causes no correctness issues unless two threads concurrently execute
473 * the arenas.create mallctl, which we trust mallctl synchronization to
477 /* Copy/initialize tickers. */
478 for (i = 0; i < narenas_actual; i++) {
479 if (i < narenas_tdata_old) {
480 ticker_copy(&arenas_tdata[i].decay_ticker,
481 &arenas_tdata_old[i].decay_ticker);
483 ticker_init(&arenas_tdata[i].decay_ticker,
484 DECAY_NTICKS_PER_UPDATE);
487 if (narenas_tdata > narenas_actual) {
488 memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t)
489 * (narenas_tdata - narenas_actual));
492 /* Read the refreshed tdata array. */
493 tdata = &arenas_tdata[ind];
495 if (arenas_tdata_old != NULL) {
496 a0dalloc(arenas_tdata_old);
501 /* Slow path, called only by arena_choose(). */
503 arena_choose_hard(tsd_t *tsd, bool internal) {
504 arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);
506 if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
507 unsigned choose = percpu_arena_choose();
508 ret = arena_get(tsd_tsdn(tsd), choose, true);
510 arena_bind(tsd, arena_ind_get(ret), false);
511 arena_bind(tsd, arena_ind_get(ret), true);
516 if (narenas_auto > 1) {
517 unsigned i, j, choose[2], first_null;
518 bool is_new_arena[2];
521 * Determine binding for both non-internal and internal
524 * choose[0]: For application allocation.
525 * choose[1]: For internal metadata allocation.
528 for (j = 0; j < 2; j++) {
530 is_new_arena[j] = false;
533 first_null = narenas_auto;
534 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
535 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
536 for (i = 1; i < narenas_auto; i++) {
537 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
539 * Choose the first arena that has the lowest
540 * number of threads assigned to it.
542 for (j = 0; j < 2; j++) {
543 if (arena_nthreads_get(arena_get(
544 tsd_tsdn(tsd), i, false), !!j) <
545 arena_nthreads_get(arena_get(
546 tsd_tsdn(tsd), choose[j], false),
551 } else if (first_null == narenas_auto) {
553 * Record the index of the first uninitialized
554 * arena, in case all extant arenas are in use.
556 * NB: It is possible for there to be
557 * discontinuities in terms of initialized
558 * versus uninitialized arenas, due to the
559 * "thread.arena" mallctl.
565 for (j = 0; j < 2; j++) {
566 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd),
567 choose[j], false), !!j) == 0 || first_null ==
570 * Use an unloaded arena, or the least loaded
571 * arena if all arenas are already initialized.
573 if (!!j == internal) {
574 ret = arena_get(tsd_tsdn(tsd),
580 /* Initialize a new arena. */
581 choose[j] = first_null;
582 arena = arena_init_locked(tsd_tsdn(tsd),
584 (extent_hooks_t *)&extent_hooks_default);
586 malloc_mutex_unlock(tsd_tsdn(tsd),
590 is_new_arena[j] = true;
591 if (!!j == internal) {
595 arena_bind(tsd, choose[j], !!j);
597 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);
599 for (j = 0; j < 2; j++) {
600 if (is_new_arena[j]) {
601 assert(choose[j] > 0);
602 arena_new_create_background_thread(
603 tsd_tsdn(tsd), choose[j]);
608 ret = arena_get(tsd_tsdn(tsd), 0, false);
609 arena_bind(tsd, 0, false);
610 arena_bind(tsd, 0, true);
617 iarena_cleanup(tsd_t *tsd) {
620 iarena = tsd_iarena_get(tsd);
621 if (iarena != NULL) {
622 arena_unbind(tsd, arena_ind_get(iarena), true);
627 arena_cleanup(tsd_t *tsd) {
630 arena = tsd_arena_get(tsd);
632 arena_unbind(tsd, arena_ind_get(arena), false);
637 arenas_tdata_cleanup(tsd_t *tsd) {
638 arena_tdata_t *arenas_tdata;
640 /* Prevent tsd->arenas_tdata from being (re)created. */
641 *tsd_arenas_tdata_bypassp_get(tsd) = true;
643 arenas_tdata = tsd_arenas_tdata_get(tsd);
644 if (arenas_tdata != NULL) {
645 tsd_arenas_tdata_set(tsd, NULL);
646 a0dalloc(arenas_tdata);
651 stats_print_atexit(void) {
659 * Merge stats from extant threads. This is racy, since
660 * individual threads do not lock when recording tcache stats
661 * events. As a consequence, the final stats may be slightly
662 * out of date by the time they are reported, if other threads
663 * continue to allocate.
665 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
666 arena_t *arena = arena_get(tsdn, i, false);
670 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
671 ql_foreach(tcache, &arena->tcache_ql, link) {
672 tcache_stats_merge(tsdn, tcache, arena);
674 malloc_mutex_unlock(tsdn,
675 &arena->tcache_ql_mtx);
679 je_malloc_stats_print(NULL, NULL, opt_stats_print_opts);
683 * Ensure that we don't hold any locks upon entry to or exit from allocator
684 * code (in a "broad" sense that doesn't count a reentrant allocation as an
687 JEMALLOC_ALWAYS_INLINE void
688 check_entry_exit_locking(tsdn_t *tsdn) {
692 if (tsdn_null(tsdn)) {
695 tsd_t *tsd = tsdn_tsd(tsdn);
697 * It's possible we hold locks at entry/exit if we're in a nested
700 int8_t reentrancy_level = tsd_reentrancy_level_get(tsd);
701 if (reentrancy_level != 0) {
704 witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
708 * End miscellaneous support functions.
710 /******************************************************************************/
712 * Begin initialization functions.
716 jemalloc_secure_getenv(const char *name) {
717 #ifdef JEMALLOC_HAVE_SECURE_GETENV
718 return secure_getenv(name);
720 # ifdef JEMALLOC_HAVE_ISSETUGID
721 if (issetugid() != 0) {
736 result = si.dwNumberOfProcessors;
737 #elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT)
739 * glibc >= 2.6 has the CPU_COUNT macro.
741 * glibc's sysconf() uses isspace(). glibc allocates for the first time
742 * *before* setting up the isspace tables. Therefore we need a
743 * different method to get the number of CPUs.
748 pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
749 result = CPU_COUNT(&set);
752 result = sysconf(_SC_NPROCESSORS_ONLN);
754 return ((result == -1) ? 1 : (unsigned)result);
758 init_opt_stats_print_opts(const char *v, size_t vlen) {
759 size_t opts_len = strlen(opt_stats_print_opts);
760 assert(opts_len <= stats_print_tot_num_options);
762 for (size_t i = 0; i < vlen; i++) {
764 #define OPTION(o, v, d, s) case o: break;
770 if (strchr(opt_stats_print_opts, v[i]) != NULL) {
771 /* Ignore repeated. */
775 opt_stats_print_opts[opts_len++] = v[i];
776 opt_stats_print_opts[opts_len] = '\0';
777 assert(opts_len <= stats_print_tot_num_options);
779 assert(opts_len == strlen(opt_stats_print_opts));
782 /* Reads the next size pair in a multi-sized option. */
784 malloc_conf_multi_sizes_next(const char **slab_size_segment_cur,
785 size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) {
786 const char *cur = *slab_size_segment_cur;
792 /* First number, then '-' */
793 um = malloc_strtoumax(cur, &end, 0);
794 if (get_errno() != 0 || *end != '-') {
797 *slab_start = (size_t)um;
800 /* Second number, then ':' */
801 um = malloc_strtoumax(cur, &end, 0);
802 if (get_errno() != 0 || *end != ':') {
805 *slab_end = (size_t)um;
809 um = malloc_strtoumax(cur, &end, 0);
810 if (get_errno() != 0) {
813 *new_size = (size_t)um;
815 /* Consume the separator if there is one. */
820 *vlen_left -= end - *slab_size_segment_cur;
821 *slab_size_segment_cur = end;
827 malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
828 char const **v_p, size_t *vlen_p) {
830 const char *opts = *opts_p;
834 for (accept = false; !accept;) {
836 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
837 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
838 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
839 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
841 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
842 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
843 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
844 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
846 case '0': case '1': case '2': case '3': case '4': case '5':
847 case '6': case '7': case '8': case '9':
853 *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
858 if (opts != *opts_p) {
859 malloc_write("<jemalloc>: Conf string ends "
864 malloc_write("<jemalloc>: Malformed conf string\n");
869 for (accept = false; !accept;) {
874 * Look ahead one character here, because the next time
875 * this function is called, it will assume that end of
876 * input has been cleanly reached if no input remains,
877 * but we have optimistically already consumed the
878 * comma if one exists.
881 malloc_write("<jemalloc>: Conf string ends "
884 *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
888 *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
902 malloc_abort_invalid_conf(void) {
903 assert(opt_abort_conf);
904 malloc_printf("<jemalloc>: Abort (abort_conf:true) on invalid conf "
905 "value (see above).\n");
910 malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
912 malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
914 /* If abort_conf is set, error out after processing all options. */
915 const char *experimental = "experimental_";
916 if (strncmp(k, experimental, strlen(experimental)) == 0) {
917 /* However, tolerate experimental features. */
920 had_conf_error = true;
924 malloc_slow_flag_init(void) {
926 * Combine the runtime options into malloc_slow for fast path. Called
927 * after processing all the options.
929 malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0)
930 | (opt_junk_free ? flag_opt_junk_free : 0)
931 | (opt_zero ? flag_opt_zero : 0)
932 | (opt_utrace ? flag_opt_utrace : 0)
933 | (opt_xmalloc ? flag_opt_xmalloc : 0);
935 malloc_slow = (malloc_slow_flags != 0);
938 /* Number of sources for initializing malloc_conf */
939 #define MALLOC_CONF_NSOURCES 4
942 obtain_malloc_conf(unsigned which_source, char buf[PATH_MAX + 1]) {
944 static unsigned read_source = 0;
946 * Each source should only be read once, to minimize # of
949 assert(read_source++ == which_source);
951 assert(which_source < MALLOC_CONF_NSOURCES);
954 switch (which_source) {
956 ret = config_malloc_conf;
959 if (je_malloc_conf != NULL) {
960 /* Use options that were compiled into the program. */
961 ret = je_malloc_conf;
963 /* No configuration specified. */
970 int saved_errno = errno;
971 const char *linkname =
972 # ifdef JEMALLOC_PREFIX
973 "/etc/"JEMALLOC_PREFIX"malloc.conf"
980 * Try to use the contents of the "/etc/malloc.conf" symbolic
983 #ifndef JEMALLOC_READLINKAT
984 linklen = readlink(linkname, buf, PATH_MAX);
986 linklen = readlinkat(AT_FDCWD, linkname, buf, PATH_MAX);
989 /* No configuration specified. */
992 set_errno(saved_errno);
999 const char *envname =
1000 #ifdef JEMALLOC_PREFIX
1001 JEMALLOC_CPREFIX"MALLOC_CONF"
1007 if ((ret = jemalloc_secure_getenv(envname)) != NULL) {
1009 * Do nothing; opts is already initialized to the value
1010 * of the MALLOC_CONF environment variable.
1013 /* No configuration specified. */
1025 malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
1026 bool initial_call, const char *opts_cache[MALLOC_CONF_NSOURCES],
1027 char buf[PATH_MAX + 1]) {
1028 static const char *opts_explain[MALLOC_CONF_NSOURCES] = {
1029 "string specified via --with-malloc-conf",
1030 "string pointed to by the global variable malloc_conf",
1031 "\"name\" of the file referenced by the symbolic link named "
1033 "value of the environment variable MALLOC_CONF"
1036 const char *opts, *k, *v;
1039 for (i = 0; i < MALLOC_CONF_NSOURCES; i++) {
1040 /* Get runtime configuration. */
1042 opts_cache[i] = obtain_malloc_conf(i, buf);
1044 opts = opts_cache[i];
1045 if (!initial_call && opt_confirm_conf) {
1047 "<jemalloc>: malloc_conf #%u (%s): \"%s\"\n",
1048 i + 1, opts_explain[i], opts != NULL ? opts : "");
1054 while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v,
1057 #define CONF_ERROR(msg, k, klen, v, vlen) \
1058 if (!initial_call) { \
1059 malloc_conf_error( \
1060 msg, k, klen, v, vlen); \
1061 cur_opt_valid = false; \
1063 #define CONF_CONTINUE { \
1064 if (!initial_call && opt_confirm_conf \
1065 && cur_opt_valid) { \
1066 malloc_printf("<jemalloc>: -- " \
1067 "Set conf value: %.*s:%.*s" \
1068 "\n", (int)klen, k, \
1073 #define CONF_MATCH(n) \
1074 (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0)
1075 #define CONF_MATCH_VALUE(n) \
1076 (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0)
1077 #define CONF_HANDLE_BOOL(o, n) \
1078 if (CONF_MATCH(n)) { \
1079 if (CONF_MATCH_VALUE("true")) { \
1081 } else if (CONF_MATCH_VALUE("false")) { \
1084 CONF_ERROR("Invalid conf value",\
1085 k, klen, v, vlen); \
1090 * One of the CONF_MIN macros below expands, in one of the use points,
1091 * to "unsigned integer < 0", which is always false, triggering the
1092 * GCC -Wtype-limits warning, which we disable here and re-enable below.
1094 JEMALLOC_DIAGNOSTIC_PUSH
1095 JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS
1097 #define CONF_DONT_CHECK_MIN(um, min) false
1098 #define CONF_CHECK_MIN(um, min) ((um) < (min))
1099 #define CONF_DONT_CHECK_MAX(um, max) false
1100 #define CONF_CHECK_MAX(um, max) ((um) > (max))
1101 #define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \
1102 if (CONF_MATCH(n)) { \
1107 um = malloc_strtoumax(v, &end, 0); \
1108 if (get_errno() != 0 || (uintptr_t)end -\
1109 (uintptr_t)v != vlen) { \
1110 CONF_ERROR("Invalid conf value",\
1111 k, klen, v, vlen); \
1112 } else if (clip) { \
1113 if (check_min(um, (t)(min))) { \
1116 check_max(um, (t)(max))) { \
1122 if (check_min(um, (t)(min)) || \
1123 check_max(um, (t)(max))) { \
1127 k, klen, v, vlen); \
1134 #define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \
1136 CONF_HANDLE_T_U(unsigned, o, n, min, max, \
1137 check_min, check_max, clip)
1138 #define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \
1139 CONF_HANDLE_T_U(size_t, o, n, min, max, \
1140 check_min, check_max, clip)
1141 #define CONF_HANDLE_SSIZE_T(o, n, min, max) \
1142 if (CONF_MATCH(n)) { \
1147 l = strtol(v, &end, 0); \
1148 if (get_errno() != 0 || (uintptr_t)end -\
1149 (uintptr_t)v != vlen) { \
1150 CONF_ERROR("Invalid conf value",\
1151 k, klen, v, vlen); \
1152 } else if (l < (ssize_t)(min) || l > \
1155 "Out-of-range conf value", \
1156 k, klen, v, vlen); \
1162 #define CONF_HANDLE_CHAR_P(o, n, d) \
1163 if (CONF_MATCH(n)) { \
1164 size_t cpylen = (vlen <= \
1165 sizeof(o)-1) ? vlen : \
1167 strncpy(o, v, cpylen); \
1172 bool cur_opt_valid = true;
1174 CONF_HANDLE_BOOL(opt_confirm_conf, "confirm_conf")
1179 CONF_HANDLE_BOOL(opt_abort, "abort")
1180 CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf")
1181 if (strncmp("metadata_thp", k, klen) == 0) {
1184 for (i = 0; i < metadata_thp_mode_limit; i++) {
1185 if (strncmp(metadata_thp_mode_names[i],
1187 opt_metadata_thp = i;
1193 CONF_ERROR("Invalid conf value",
1198 CONF_HANDLE_BOOL(opt_retain, "retain")
1199 if (strncmp("dss", k, klen) == 0) {
1202 for (i = 0; i < dss_prec_limit; i++) {
1203 if (strncmp(dss_prec_names[i], v, vlen)
1205 if (extent_dss_prec_set(i)) {
1207 "Error setting dss",
1218 CONF_ERROR("Invalid conf value",
1223 CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
1224 UINT_MAX, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX,
1226 if (CONF_MATCH("bin_shards")) {
1227 const char *bin_shards_segment_cur = v;
1228 size_t vlen_left = vlen;
1233 bool err = malloc_conf_multi_sizes_next(
1234 &bin_shards_segment_cur, &vlen_left,
1235 &size_start, &size_end, &nshards);
1236 if (err || bin_update_shard_size(
1237 bin_shard_sizes, size_start,
1238 size_end, nshards)) {
1240 "Invalid settings for "
1241 "bin_shards", k, klen, v,
1245 } while (vlen_left > 0);
1248 CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms,
1249 "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
1250 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
1252 CONF_HANDLE_SSIZE_T(opt_muzzy_decay_ms,
1253 "muzzy_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
1254 QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
1256 CONF_HANDLE_BOOL(opt_stats_print, "stats_print")
1257 if (CONF_MATCH("stats_print_opts")) {
1258 init_opt_stats_print_opts(v, vlen);
1262 if (CONF_MATCH("junk")) {
1263 if (CONF_MATCH_VALUE("true")) {
1265 opt_junk_alloc = opt_junk_free =
1267 } else if (CONF_MATCH_VALUE("false")) {
1269 opt_junk_alloc = opt_junk_free =
1271 } else if (CONF_MATCH_VALUE("alloc")) {
1273 opt_junk_alloc = true;
1274 opt_junk_free = false;
1275 } else if (CONF_MATCH_VALUE("free")) {
1277 opt_junk_alloc = false;
1278 opt_junk_free = true;
1281 "Invalid conf value",
1286 CONF_HANDLE_BOOL(opt_zero, "zero")
1288 if (config_utrace) {
1289 CONF_HANDLE_BOOL(opt_utrace, "utrace")
1291 if (config_xmalloc) {
1292 CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
1294 CONF_HANDLE_BOOL(opt_tcache, "tcache")
1295 CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max",
1296 -1, (sizeof(size_t) << 3) - 1)
1299 * The runtime option of oversize_threshold remains
1300 * undocumented. It may be tweaked in the next major
1301 * release (6.0). The default value 8M is rather
1302 * conservative / safe. Tuning it further down may
1303 * improve fragmentation a bit more, but may also cause
1304 * contention on the huge arena.
1306 CONF_HANDLE_SIZE_T(opt_oversize_threshold,
1307 "oversize_threshold", 0, SC_LARGE_MAXCLASS,
1308 CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, false)
1309 CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit,
1310 "lg_extent_max_active_fit", 0,
1311 (sizeof(size_t) << 3), CONF_DONT_CHECK_MIN,
1312 CONF_CHECK_MAX, false)
1314 if (strncmp("percpu_arena", k, klen) == 0) {
1316 for (int i = percpu_arena_mode_names_base; i <
1317 percpu_arena_mode_names_limit; i++) {
1318 if (strncmp(percpu_arena_mode_names[i],
1320 if (!have_percpu_arena) {
1322 "No getcpu support",
1325 opt_percpu_arena = i;
1331 CONF_ERROR("Invalid conf value",
1336 CONF_HANDLE_BOOL(opt_background_thread,
1337 "background_thread");
1338 CONF_HANDLE_SIZE_T(opt_max_background_threads,
1339 "max_background_threads", 1,
1340 opt_max_background_threads,
1341 CONF_CHECK_MIN, CONF_CHECK_MAX,
1343 if (CONF_MATCH("slab_sizes")) {
1345 const char *slab_size_segment_cur = v;
1346 size_t vlen_left = vlen;
1351 err = malloc_conf_multi_sizes_next(
1352 &slab_size_segment_cur,
1353 &vlen_left, &slab_start, &slab_end,
1356 sc_data_update_slab_size(
1357 sc_data, slab_start,
1358 slab_end, (int)pgs);
1360 CONF_ERROR("Invalid settings "
1364 } while (!err && vlen_left > 0);
1368 CONF_HANDLE_BOOL(opt_prof, "prof")
1369 CONF_HANDLE_CHAR_P(opt_prof_prefix,
1370 "prof_prefix", "jeprof")
1371 CONF_HANDLE_BOOL(opt_prof_active, "prof_active")
1372 CONF_HANDLE_BOOL(opt_prof_thread_active_init,
1373 "prof_thread_active_init")
1374 CONF_HANDLE_SIZE_T(opt_lg_prof_sample,
1375 "lg_prof_sample", 0, (sizeof(uint64_t) << 3)
1376 - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
1378 CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum")
1379 CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
1380 "lg_prof_interval", -1,
1381 (sizeof(uint64_t) << 3) - 1)
1382 CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump")
1383 CONF_HANDLE_BOOL(opt_prof_final, "prof_final")
1384 CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak")
1385 CONF_HANDLE_BOOL(opt_prof_log, "prof_log")
1388 if (CONF_MATCH("log")) {
1390 vlen <= sizeof(log_var_names) ?
1391 vlen : sizeof(log_var_names) - 1);
1392 strncpy(log_var_names, v, cpylen);
1393 log_var_names[cpylen] = '\0';
1397 if (CONF_MATCH("thp")) {
1399 for (int i = 0; i < thp_mode_names_limit; i++) {
1400 if (strncmp(thp_mode_names[i],v, vlen)
1402 if (!have_madvise_huge) {
1413 CONF_ERROR("Invalid conf value",
1418 CONF_ERROR("Invalid conf pair", k, klen, v, vlen);
1420 #undef CONF_CONTINUE
1422 #undef CONF_MATCH_VALUE
1423 #undef CONF_HANDLE_BOOL
1424 #undef CONF_DONT_CHECK_MIN
1425 #undef CONF_CHECK_MIN
1426 #undef CONF_DONT_CHECK_MAX
1427 #undef CONF_CHECK_MAX
1428 #undef CONF_HANDLE_T_U
1429 #undef CONF_HANDLE_UNSIGNED
1430 #undef CONF_HANDLE_SIZE_T
1431 #undef CONF_HANDLE_SSIZE_T
1432 #undef CONF_HANDLE_CHAR_P
1433 /* Re-enable diagnostic "-Wtype-limits" */
1434 JEMALLOC_DIAGNOSTIC_POP
1436 if (opt_abort_conf && had_conf_error) {
1437 malloc_abort_invalid_conf();
1440 atomic_store_b(&log_init_done, true, ATOMIC_RELEASE);
1444 malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
1445 const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL};
1446 char buf[PATH_MAX + 1];
1448 /* The first call only set the confirm_conf option and opts_cache */
1449 malloc_conf_init_helper(NULL, NULL, true, opts_cache, buf);
1450 malloc_conf_init_helper(sc_data, bin_shard_sizes, false, opts_cache,
1454 #undef MALLOC_CONF_NSOURCES
1457 malloc_init_hard_needed(void) {
1458 if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state ==
1459 malloc_init_recursible)) {
1461 * Another thread initialized the allocator before this one
1462 * acquired init_lock, or this thread is the initializing
1463 * thread, and it is recursively allocating.
1467 #ifdef JEMALLOC_THREADED_INIT
1468 if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
1469 /* Busy-wait until the initializing thread completes. */
1470 spin_t spinner = SPIN_INITIALIZER;
1472 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1473 spin_adaptive(&spinner);
1474 malloc_mutex_lock(TSDN_NULL, &init_lock);
1475 } while (!malloc_initialized());
1483 malloc_init_hard_a0_locked() {
1484 malloc_initializer = INITIALIZER;
1486 JEMALLOC_DIAGNOSTIC_PUSH
1487 JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
1488 sc_data_t sc_data = {0};
1489 JEMALLOC_DIAGNOSTIC_POP
1492 * Ordering here is somewhat tricky; we need sc_boot() first, since that
1493 * determines what the size classes will be, and then
1494 * malloc_conf_init(), since any slab size tweaking will need to be done
1495 * before sz_boot and bin_boot, which assume that the values they read
1496 * out of sc_data_global are final.
1499 unsigned bin_shard_sizes[SC_NBINS];
1500 bin_shard_sizes_boot(bin_shard_sizes);
1502 * prof_boot0 only initializes opt_prof_prefix. We need to do it before
1503 * we parse malloc_conf options, in case malloc_conf parsing overwrites
1509 malloc_conf_init(&sc_data, bin_shard_sizes);
1511 bin_boot(&sc_data, bin_shard_sizes);
1513 if (opt_stats_print) {
1514 /* Print statistics at exit. */
1515 if (atexit(stats_print_atexit) != 0) {
1516 malloc_write("<jemalloc>: Error in atexit()\n");
1525 if (base_boot(TSDN_NULL)) {
1528 if (extent_boot()) {
1537 arena_boot(&sc_data);
1538 if (tcache_boot(TSDN_NULL)) {
1541 if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS,
1542 malloc_mutex_rank_exclusive)) {
1547 * Create enough scaffolding to allow recursive allocation in
1551 manual_arena_base = narenas_auto + 1;
1552 memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
1554 * Initialize one arena here. The rest are lazily created in
1555 * arena_choose_hard().
1557 if (arena_init(TSDN_NULL, 0, (extent_hooks_t *)&extent_hooks_default)
1561 a0 = arena_get(TSDN_NULL, 0, false);
1562 malloc_init_state = malloc_init_a0_initialized;
1568 malloc_init_hard_a0(void) {
1571 malloc_mutex_lock(TSDN_NULL, &init_lock);
1572 ret = malloc_init_hard_a0_locked();
1573 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1577 /* Initialize data structures which may trigger recursive allocation. */
1579 malloc_init_hard_recursible(void) {
1580 malloc_init_state = malloc_init_recursible;
1582 ncpus = malloc_ncpus();
1584 #if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
1585 && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \
1586 !defined(__native_client__))
1587 /* LinuxThreads' pthread_atfork() allocates. */
1588 if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
1589 jemalloc_postfork_child) != 0) {
1590 malloc_write("<jemalloc>: Error in pthread_atfork()\n");
1598 if (background_thread_boot0()) {
1606 malloc_narenas_default(void) {
1609 * For SMP systems, create more than one arena per CPU by
1619 static percpu_arena_mode_t
1620 percpu_arena_as_initialized(percpu_arena_mode_t mode) {
1621 assert(!malloc_initialized());
1622 assert(mode <= percpu_arena_disabled);
1624 if (mode != percpu_arena_disabled) {
1625 mode += percpu_arena_mode_enabled_base;
1632 malloc_init_narenas(void) {
1635 if (opt_percpu_arena != percpu_arena_disabled) {
1636 if (!have_percpu_arena || malloc_getcpu() < 0) {
1637 opt_percpu_arena = percpu_arena_disabled;
1638 malloc_printf("<jemalloc>: perCPU arena getcpu() not "
1639 "available. Setting narenas to %u.\n", opt_narenas ?
1640 opt_narenas : malloc_narenas_default());
1645 if (ncpus >= MALLOCX_ARENA_LIMIT) {
1646 malloc_printf("<jemalloc>: narenas w/ percpu"
1647 "arena beyond limit (%d)\n", ncpus);
1653 /* NB: opt_percpu_arena isn't fully initialized yet. */
1654 if (percpu_arena_as_initialized(opt_percpu_arena) ==
1655 per_phycpu_arena && ncpus % 2 != 0) {
1656 malloc_printf("<jemalloc>: invalid "
1657 "configuration -- per physical CPU arena "
1658 "with odd number (%u) of CPUs (no hyper "
1659 "threading?).\n", ncpus);
1663 unsigned n = percpu_arena_ind_limit(
1664 percpu_arena_as_initialized(opt_percpu_arena));
1665 if (opt_narenas < n) {
1667 * If narenas is specified with percpu_arena
1668 * enabled, actual narenas is set as the greater
1669 * of the two. percpu_arena_choose will be free
1670 * to use any of the arenas based on CPU
1671 * id. This is conservative (at a small cost)
1672 * but ensures correctness.
1674 * If for some reason the ncpus determined at
1675 * boot is not the actual number (e.g. because
1676 * of affinity setting from numactl), reserving
1677 * narenas this way provides a workaround for
1684 if (opt_narenas == 0) {
1685 opt_narenas = malloc_narenas_default();
1687 assert(opt_narenas > 0);
1689 narenas_auto = opt_narenas;
1691 * Limit the number of arenas to the indexing range of MALLOCX_ARENA().
1693 if (narenas_auto >= MALLOCX_ARENA_LIMIT) {
1694 narenas_auto = MALLOCX_ARENA_LIMIT - 1;
1695 malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
1698 narenas_total_set(narenas_auto);
1699 if (arena_init_huge()) {
1700 narenas_total_inc();
1702 manual_arena_base = narenas_total_get();
1708 malloc_init_percpu(void) {
1709 opt_percpu_arena = percpu_arena_as_initialized(opt_percpu_arena);
1713 malloc_init_hard_finish(void) {
1714 if (malloc_mutex_boot()) {
1718 malloc_init_state = malloc_init_initialized;
1719 malloc_slow_flag_init();
1725 malloc_init_hard_cleanup(tsdn_t *tsdn, bool reentrancy_set) {
1726 malloc_mutex_assert_owner(tsdn, &init_lock);
1727 malloc_mutex_unlock(tsdn, &init_lock);
1728 if (reentrancy_set) {
1729 assert(!tsdn_null(tsdn));
1730 tsd_t *tsd = tsdn_tsd(tsdn);
1731 assert(tsd_reentrancy_level_get(tsd) > 0);
1732 post_reentrancy(tsd);
1737 malloc_init_hard(void) {
1740 #if defined(_WIN32) && _WIN32_WINNT < 0x0600
1743 malloc_mutex_lock(TSDN_NULL, &init_lock);
1745 #define UNLOCK_RETURN(tsdn, ret, reentrancy) \
1746 malloc_init_hard_cleanup(tsdn, reentrancy); \
1749 if (!malloc_init_hard_needed()) {
1750 UNLOCK_RETURN(TSDN_NULL, false, false)
1753 if (malloc_init_state != malloc_init_a0_initialized &&
1754 malloc_init_hard_a0_locked()) {
1755 UNLOCK_RETURN(TSDN_NULL, true, false)
1758 malloc_mutex_unlock(TSDN_NULL, &init_lock);
1759 /* Recursive allocation relies on functional tsd. */
1760 tsd = malloc_tsd_boot0();
1764 if (malloc_init_hard_recursible()) {
1768 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
1769 /* Set reentrancy level to 1 during init. */
1770 pre_reentrancy(tsd, NULL);
1771 /* Initialize narenas before prof_boot2 (for allocation). */
1772 if (malloc_init_narenas() || background_thread_boot1(tsd_tsdn(tsd))) {
1773 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
1775 if (config_prof && prof_boot2(tsd)) {
1776 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
1779 malloc_init_percpu();
1781 if (malloc_init_hard_finish()) {
1782 UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
1784 post_reentrancy(tsd);
1785 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
1787 witness_assert_lockless(witness_tsd_tsdn(
1788 tsd_witness_tsdp_get_unsafe(tsd)));
1790 /* Update TSD after tsd_boot1. */
1792 if (opt_background_thread) {
1793 assert(have_background_thread);
1795 * Need to finish init & unlock first before creating background
1796 * threads (pthread_create depends on malloc). ctl_init (which
1797 * sets isthreaded) needs to be called without holding any lock.
1799 background_thread_ctl_init(tsd_tsdn(tsd));
1800 if (background_thread_create(tsd, 0)) {
1804 #undef UNLOCK_RETURN
1809 * End initialization functions.
1811 /******************************************************************************/
1813 * Begin allocation-path internal functions and data structures.
1817 * Settings determined by the documented behavior of the allocation functions.
1819 typedef struct static_opts_s static_opts_t;
1820 struct static_opts_s {
1821 /* Whether or not allocation size may overflow. */
1825 * Whether or not allocations (with alignment) of size 0 should be
1826 * treated as size 1.
1828 bool bump_empty_aligned_alloc;
1830 * Whether to assert that allocations are not of size 0 (after any
1833 bool assert_nonempty_alloc;
1836 * Whether or not to modify the 'result' argument to malloc in case of
1839 bool null_out_result_on_error;
1840 /* Whether to set errno when we encounter an error condition. */
1841 bool set_errno_on_error;
1844 * The minimum valid alignment for functions requesting aligned storage.
1846 size_t min_alignment;
1848 /* The error string to use if we oom. */
1849 const char *oom_string;
1850 /* The error string to use if the passed-in alignment is invalid. */
1851 const char *invalid_alignment_string;
1854 * False if we're configured to skip some time-consuming operations.
1856 * This isn't really a malloc "behavior", but it acts as a useful
1857 * summary of several other static (or at least, static after program
1858 * initialization) options.
1867 JEMALLOC_ALWAYS_INLINE void
1868 static_opts_init(static_opts_t *static_opts) {
1869 static_opts->may_overflow = false;
1870 static_opts->bump_empty_aligned_alloc = false;
1871 static_opts->assert_nonempty_alloc = false;
1872 static_opts->null_out_result_on_error = false;
1873 static_opts->set_errno_on_error = false;
1874 static_opts->min_alignment = 0;
1875 static_opts->oom_string = "";
1876 static_opts->invalid_alignment_string = "";
1877 static_opts->slow = false;
1878 static_opts->usize = false;
1882 * These correspond to the macros in jemalloc/jemalloc_macros.h. Broadly, we
1883 * should have one constant here per magic value there. Note however that the
1884 * representations need not be related.
1886 #define TCACHE_IND_NONE ((unsigned)-1)
1887 #define TCACHE_IND_AUTOMATIC ((unsigned)-2)
1888 #define ARENA_IND_AUTOMATIC ((unsigned)-1)
1890 typedef struct dynamic_opts_s dynamic_opts_t;
1891 struct dynamic_opts_s {
1898 unsigned tcache_ind;
1902 JEMALLOC_ALWAYS_INLINE void
1903 dynamic_opts_init(dynamic_opts_t *dynamic_opts) {
1904 dynamic_opts->result = NULL;
1905 dynamic_opts->usize = 0;
1906 dynamic_opts->num_items = 0;
1907 dynamic_opts->item_size = 0;
1908 dynamic_opts->alignment = 0;
1909 dynamic_opts->zero = false;
1910 dynamic_opts->tcache_ind = TCACHE_IND_AUTOMATIC;
1911 dynamic_opts->arena_ind = ARENA_IND_AUTOMATIC;
1914 /* ind is ignored if dopts->alignment > 0. */
1915 JEMALLOC_ALWAYS_INLINE void *
1916 imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
1917 size_t size, size_t usize, szind_t ind) {
1921 /* Fill in the tcache. */
1922 if (dopts->tcache_ind == TCACHE_IND_AUTOMATIC) {
1923 if (likely(!sopts->slow)) {
1924 /* Getting tcache ptr unconditionally. */
1925 tcache = tsd_tcachep_get(tsd);
1926 assert(tcache == tcache_get(tsd));
1928 tcache = tcache_get(tsd);
1930 } else if (dopts->tcache_ind == TCACHE_IND_NONE) {
1933 tcache = tcaches_get(tsd, dopts->tcache_ind);
1936 /* Fill in the arena. */
1937 if (dopts->arena_ind == ARENA_IND_AUTOMATIC) {
1939 * In case of automatic arena management, we defer arena
1940 * computation until as late as we can, hoping to fill the
1941 * allocation out of the tcache.
1945 arena = arena_get(tsd_tsdn(tsd), dopts->arena_ind, true);
1948 if (unlikely(dopts->alignment != 0)) {
1949 return ipalloct(tsd_tsdn(tsd), usize, dopts->alignment,
1950 dopts->zero, tcache, arena);
1953 return iallocztm(tsd_tsdn(tsd), size, ind, dopts->zero, tcache, false,
1954 arena, sopts->slow);
1957 JEMALLOC_ALWAYS_INLINE void *
1958 imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
1959 size_t usize, szind_t ind) {
1963 * For small allocations, sampling bumps the usize. If so, we allocate
1964 * from the ind_large bucket.
1967 size_t bumped_usize = usize;
1969 if (usize <= SC_SMALL_MAXCLASS) {
1970 assert(((dopts->alignment == 0) ?
1971 sz_s2u(SC_LARGE_MINCLASS) :
1972 sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment))
1973 == SC_LARGE_MINCLASS);
1974 ind_large = sz_size2index(SC_LARGE_MINCLASS);
1975 bumped_usize = sz_s2u(SC_LARGE_MINCLASS);
1976 ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize,
1977 bumped_usize, ind_large);
1978 if (unlikely(ret == NULL)) {
1981 arena_prof_promote(tsd_tsdn(tsd), ret, usize);
1983 ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind);
1990 * Returns true if the allocation will overflow, and false otherwise. Sets
1991 * *size to the product either way.
1993 JEMALLOC_ALWAYS_INLINE bool
1994 compute_size_with_overflow(bool may_overflow, dynamic_opts_t *dopts,
1997 * This function is just num_items * item_size, except that we may have
1998 * to check for overflow.
2001 if (!may_overflow) {
2002 assert(dopts->num_items == 1);
2003 *size = dopts->item_size;
2007 /* A size_t with its high-half bits all set to 1. */
2008 static const size_t high_bits = SIZE_T_MAX << (sizeof(size_t) * 8 / 2);
2010 *size = dopts->item_size * dopts->num_items;
2012 if (unlikely(*size == 0)) {
2013 return (dopts->num_items != 0 && dopts->item_size != 0);
2017 * We got a non-zero size, but we don't know if we overflowed to get
2018 * there. To avoid having to do a divide, we'll be clever and note that
2019 * if both A and B can be represented in N/2 bits, then their product
2020 * can be represented in N bits (without the possibility of overflow).
2022 if (likely((high_bits & (dopts->num_items | dopts->item_size)) == 0)) {
2025 if (likely(*size / dopts->item_size == dopts->num_items)) {
2031 JEMALLOC_ALWAYS_INLINE int
2032 imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
2033 /* Where the actual allocated memory will live. */
2034 void *allocation = NULL;
2035 /* Filled in by compute_size_with_overflow below. */
2038 * For unaligned allocations, we need only ind. For aligned
2039 * allocations, or in case of stats or profiling we need usize.
2041 * These are actually dead stores, in that their values are reset before
2042 * any branch on their value is taken. Sometimes though, it's
2043 * convenient to pass them as arguments before this point. To avoid
2044 * undefined behavior then, we initialize them with dummy stores.
2049 /* Reentrancy is only checked on slow path. */
2050 int8_t reentrancy_level;
2052 /* Compute the amount of memory the user wants. */
2053 if (unlikely(compute_size_with_overflow(sopts->may_overflow, dopts,
2058 if (unlikely(dopts->alignment < sopts->min_alignment
2059 || (dopts->alignment & (dopts->alignment - 1)) != 0)) {
2060 goto label_invalid_alignment;
2063 /* This is the beginning of the "core" algorithm. */
2065 if (dopts->alignment == 0) {
2066 ind = sz_size2index(size);
2067 if (unlikely(ind >= SC_NSIZES)) {
2070 if (config_stats || (config_prof && opt_prof) || sopts->usize) {
2071 usize = sz_index2size(ind);
2072 dopts->usize = usize;
2073 assert(usize > 0 && usize
2074 <= SC_LARGE_MAXCLASS);
2077 if (sopts->bump_empty_aligned_alloc) {
2078 if (unlikely(size == 0)) {
2082 usize = sz_sa2u(size, dopts->alignment);
2083 dopts->usize = usize;
2084 if (unlikely(usize == 0
2085 || usize > SC_LARGE_MAXCLASS)) {
2089 /* Validate the user input. */
2090 if (sopts->assert_nonempty_alloc) {
2094 check_entry_exit_locking(tsd_tsdn(tsd));
2097 * If we need to handle reentrancy, we can do it out of a
2098 * known-initialized arena (i.e. arena 0).
2100 reentrancy_level = tsd_reentrancy_level_get(tsd);
2101 if (sopts->slow && unlikely(reentrancy_level > 0)) {
2103 * We should never specify particular arenas or tcaches from
2104 * within our internal allocations.
2106 assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC ||
2107 dopts->tcache_ind == TCACHE_IND_NONE);
2108 assert(dopts->arena_ind == ARENA_IND_AUTOMATIC);
2109 dopts->tcache_ind = TCACHE_IND_NONE;
2110 /* We know that arena 0 has already been initialized. */
2111 dopts->arena_ind = 0;
2114 /* If profiling is on, get our profiling context. */
2115 if (config_prof && opt_prof) {
2117 * Note that if we're going down this path, usize must have been
2118 * initialized in the previous if statement.
2120 prof_tctx_t *tctx = prof_alloc_prep(
2121 tsd, usize, prof_active_get_unlocked(), true);
2123 alloc_ctx_t alloc_ctx;
2124 if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
2125 alloc_ctx.slab = (usize
2126 <= SC_SMALL_MAXCLASS);
2127 allocation = imalloc_no_sample(
2128 sopts, dopts, tsd, usize, usize, ind);
2129 } else if ((uintptr_t)tctx > (uintptr_t)1U) {
2131 * Note that ind might still be 0 here. This is fine;
2132 * imalloc_sample ignores ind if dopts->alignment > 0.
2134 allocation = imalloc_sample(
2135 sopts, dopts, tsd, usize, ind);
2136 alloc_ctx.slab = false;
2141 if (unlikely(allocation == NULL)) {
2142 prof_alloc_rollback(tsd, tctx, true);
2145 prof_malloc(tsd_tsdn(tsd), allocation, usize, &alloc_ctx, tctx);
2148 * If dopts->alignment > 0, then ind is still 0, but usize was
2149 * computed in the previous if statement. Down the positive
2150 * alignment path, imalloc_no_sample ignores ind and size
2151 * (relying only on usize).
2153 allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
2155 if (unlikely(allocation == NULL)) {
2161 * Allocation has been done at this point. We still have some
2162 * post-allocation work to do though.
2164 assert(dopts->alignment == 0
2165 || ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0));
2168 assert(usize == isalloc(tsd_tsdn(tsd), allocation));
2169 *tsd_thread_allocatedp_get(tsd) += usize;
2173 UTRACE(0, size, allocation);
2177 check_entry_exit_locking(tsd_tsdn(tsd));
2178 *dopts->result = allocation;
2182 if (unlikely(sopts->slow) && config_xmalloc && unlikely(opt_xmalloc)) {
2183 malloc_write(sopts->oom_string);
2188 UTRACE(NULL, size, NULL);
2191 check_entry_exit_locking(tsd_tsdn(tsd));
2193 if (sopts->set_errno_on_error) {
2197 if (sopts->null_out_result_on_error) {
2198 *dopts->result = NULL;
2204 * This label is only jumped to by one goto; we move it out of line
2205 * anyways to avoid obscuring the non-error paths, and for symmetry with
2208 label_invalid_alignment:
2209 if (config_xmalloc && unlikely(opt_xmalloc)) {
2210 malloc_write(sopts->invalid_alignment_string);
2214 if (sopts->set_errno_on_error) {
2219 UTRACE(NULL, size, NULL);
2222 check_entry_exit_locking(tsd_tsdn(tsd));
2224 if (sopts->null_out_result_on_error) {
2225 *dopts->result = NULL;
2231 JEMALLOC_ALWAYS_INLINE bool
2232 imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) {
2233 if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) {
2234 if (config_xmalloc && unlikely(opt_xmalloc)) {
2235 malloc_write(sopts->oom_string);
2238 UTRACE(NULL, dopts->num_items * dopts->item_size, NULL);
2240 *dopts->result = NULL;
2248 /* Returns the errno-style error code of the allocation. */
2249 JEMALLOC_ALWAYS_INLINE int
2250 imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) {
2251 if (tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) {
2255 /* We always need the tsd. Let's grab it right away. */
2256 tsd_t *tsd = tsd_fetch();
2258 if (likely(tsd_fast(tsd))) {
2259 /* Fast and common path. */
2260 tsd_assert_fast(tsd);
2261 sopts->slow = false;
2262 return imalloc_body(sopts, dopts, tsd);
2264 if (!tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) {
2269 return imalloc_body(sopts, dopts, tsd);
2275 malloc_default(size_t size) {
2277 static_opts_t sopts;
2278 dynamic_opts_t dopts;
2280 LOG("core.malloc.entry", "size: %zu", size);
2282 static_opts_init(&sopts);
2283 dynamic_opts_init(&dopts);
2285 sopts.null_out_result_on_error = true;
2286 sopts.set_errno_on_error = true;
2287 sopts.oom_string = "<jemalloc>: Error in malloc(): out of memory\n";
2289 dopts.result = &ret;
2290 dopts.num_items = 1;
2291 dopts.item_size = size;
2293 imalloc(&sopts, &dopts);
2295 * Note that this branch gets optimized away -- it immediately follows
2296 * the check on tsd_fast that sets sopts.slow.
2299 uintptr_t args[3] = {size};
2300 hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args);
2303 LOG("core.malloc.exit", "result: %p", ret);
2308 /******************************************************************************/
2310 * Begin malloc(3)-compatible functions.
2314 * malloc() fastpath.
2316 * Fastpath assumes size <= SC_LOOKUP_MAXCLASS, and that we hit
2317 * tcache. If either of these is false, we tail-call to the slowpath,
2318 * malloc_default(). Tail-calling is used to avoid any caller-saved
2321 * fastpath supports ticker and profiling, both of which will also
2322 * tail-call to the slowpath if they fire.
2324 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2325 void JEMALLOC_NOTHROW *
2326 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
2327 je_malloc(size_t size) {
2328 LOG("core.malloc.entry", "size: %zu", size);
2330 if (tsd_get_allocates() && unlikely(!malloc_initialized())) {
2331 return malloc_default(size);
2334 tsd_t *tsd = tsd_get(false);
2335 if (unlikely(!tsd || !tsd_fast(tsd) || (size > SC_LOOKUP_MAXCLASS))) {
2336 return malloc_default(size);
2339 tcache_t *tcache = tsd_tcachep_get(tsd);
2341 if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
2342 return malloc_default(size);
2345 szind_t ind = sz_size2index_lookup(size);
2347 if (config_stats || config_prof) {
2348 usize = sz_index2size(ind);
2350 /* Fast path relies on size being a bin. I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS */
2351 assert(ind < SC_NBINS);
2352 assert(size <= SC_SMALL_MAXCLASS);
2355 int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd);
2356 bytes_until_sample -= usize;
2357 tsd_bytes_until_sample_set(tsd, bytes_until_sample);
2359 if (unlikely(bytes_until_sample < 0)) {
2361 * Avoid a prof_active check on the fastpath.
2362 * If prof_active is false, set bytes_until_sample to
2363 * a large value. If prof_active is set to true,
2364 * bytes_until_sample will be reset.
2367 tsd_bytes_until_sample_set(tsd, SSIZE_MAX);
2369 return malloc_default(size);
2373 cache_bin_t *bin = tcache_small_bin_get(tcache, ind);
2374 bool tcache_success;
2375 void* ret = cache_bin_alloc_easy(bin, &tcache_success);
2377 if (tcache_success) {
2379 *tsd_thread_allocatedp_get(tsd) += usize;
2380 bin->tstats.nrequests++;
2383 tcache->prof_accumbytes += usize;
2386 LOG("core.malloc.exit", "result: %p", ret);
2388 /* Fastpath success */
2392 return malloc_default(size);
2395 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
2396 JEMALLOC_ATTR(nonnull(1))
2397 je_posix_memalign(void **memptr, size_t alignment, size_t size) {
2399 static_opts_t sopts;
2400 dynamic_opts_t dopts;
2402 LOG("core.posix_memalign.entry", "mem ptr: %p, alignment: %zu, "
2403 "size: %zu", memptr, alignment, size);
2405 static_opts_init(&sopts);
2406 dynamic_opts_init(&dopts);
2408 sopts.bump_empty_aligned_alloc = true;
2409 sopts.min_alignment = sizeof(void *);
2411 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2412 sopts.invalid_alignment_string =
2413 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2415 dopts.result = memptr;
2416 dopts.num_items = 1;
2417 dopts.item_size = size;
2418 dopts.alignment = alignment;
2420 ret = imalloc(&sopts, &dopts);
2422 uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment,
2424 hook_invoke_alloc(hook_alloc_posix_memalign, *memptr,
2425 (uintptr_t)ret, args);
2428 LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret,
2434 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2435 void JEMALLOC_NOTHROW *
2436 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2)
2437 je_aligned_alloc(size_t alignment, size_t size) {
2440 static_opts_t sopts;
2441 dynamic_opts_t dopts;
2443 LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n",
2446 static_opts_init(&sopts);
2447 dynamic_opts_init(&dopts);
2449 sopts.bump_empty_aligned_alloc = true;
2450 sopts.null_out_result_on_error = true;
2451 sopts.set_errno_on_error = true;
2452 sopts.min_alignment = 1;
2454 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2455 sopts.invalid_alignment_string =
2456 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2458 dopts.result = &ret;
2459 dopts.num_items = 1;
2460 dopts.item_size = size;
2461 dopts.alignment = alignment;
2463 imalloc(&sopts, &dopts);
2465 uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size};
2466 hook_invoke_alloc(hook_alloc_aligned_alloc, ret,
2467 (uintptr_t)ret, args);
2470 LOG("core.aligned_alloc.exit", "result: %p", ret);
2475 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2476 void JEMALLOC_NOTHROW *
2477 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2)
2478 je_calloc(size_t num, size_t size) {
2480 static_opts_t sopts;
2481 dynamic_opts_t dopts;
2483 LOG("core.calloc.entry", "num: %zu, size: %zu\n", num, size);
2485 static_opts_init(&sopts);
2486 dynamic_opts_init(&dopts);
2488 sopts.may_overflow = true;
2489 sopts.null_out_result_on_error = true;
2490 sopts.set_errno_on_error = true;
2491 sopts.oom_string = "<jemalloc>: Error in calloc(): out of memory\n";
2493 dopts.result = &ret;
2494 dopts.num_items = num;
2495 dopts.item_size = size;
2498 imalloc(&sopts, &dopts);
2500 uintptr_t args[3] = {(uintptr_t)num, (uintptr_t)size};
2501 hook_invoke_alloc(hook_alloc_calloc, ret, (uintptr_t)ret, args);
2504 LOG("core.calloc.exit", "result: %p", ret);
2510 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
2511 prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) {
2517 if (usize <= SC_SMALL_MAXCLASS) {
2518 p = iralloc(tsd, old_ptr, old_usize,
2519 SC_LARGE_MINCLASS, 0, false, hook_args);
2523 arena_prof_promote(tsd_tsdn(tsd), p, usize);
2525 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false,
2532 JEMALLOC_ALWAYS_INLINE void *
2533 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
2534 alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) {
2537 prof_tctx_t *old_tctx, *tctx;
2539 prof_active = prof_active_get_unlocked();
2540 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx);
2541 tctx = prof_alloc_prep(tsd, usize, prof_active, true);
2542 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
2543 p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx,
2546 p = iralloc(tsd, old_ptr, old_usize, usize, 0, false,
2549 if (unlikely(p == NULL)) {
2550 prof_alloc_rollback(tsd, tctx, true);
2553 prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
2559 JEMALLOC_ALWAYS_INLINE void
2560 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
2562 tsd_assert_fast(tsd);
2564 check_entry_exit_locking(tsd_tsdn(tsd));
2565 if (tsd_reentrancy_level_get(tsd) != 0) {
2569 assert(ptr != NULL);
2570 assert(malloc_initialized() || IS_INITIALIZER);
2572 alloc_ctx_t alloc_ctx;
2573 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2574 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
2575 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
2576 assert(alloc_ctx.szind != SC_NSIZES);
2579 if (config_prof && opt_prof) {
2580 usize = sz_index2size(alloc_ctx.szind);
2581 prof_free(tsd, ptr, usize, &alloc_ctx);
2582 } else if (config_stats) {
2583 usize = sz_index2size(alloc_ctx.szind);
2586 *tsd_thread_deallocatedp_get(tsd) += usize;
2589 if (likely(!slow_path)) {
2590 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
2593 idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
2598 JEMALLOC_ALWAYS_INLINE void
2599 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
2601 tsd_assert_fast(tsd);
2603 check_entry_exit_locking(tsd_tsdn(tsd));
2604 if (tsd_reentrancy_level_get(tsd) != 0) {
2608 assert(ptr != NULL);
2609 assert(malloc_initialized() || IS_INITIALIZER);
2611 alloc_ctx_t alloc_ctx, *ctx;
2612 if (!config_cache_oblivious && ((uintptr_t)ptr & PAGE_MASK) != 0) {
2614 * When cache_oblivious is disabled and ptr is not page aligned,
2615 * the allocation was not sampled -- usize can be used to
2616 * determine szind directly.
2618 alloc_ctx.szind = sz_size2index(usize);
2619 alloc_ctx.slab = true;
2622 alloc_ctx_t dbg_ctx;
2623 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2624 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree,
2625 rtree_ctx, (uintptr_t)ptr, true, &dbg_ctx.szind,
2627 assert(dbg_ctx.szind == alloc_ctx.szind);
2628 assert(dbg_ctx.slab == alloc_ctx.slab);
2630 } else if (config_prof && opt_prof) {
2631 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2632 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
2633 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
2634 assert(alloc_ctx.szind == sz_size2index(usize));
2640 if (config_prof && opt_prof) {
2641 prof_free(tsd, ptr, usize, ctx);
2644 *tsd_thread_deallocatedp_get(tsd) += usize;
2647 if (likely(!slow_path)) {
2648 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false);
2650 isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true);
2654 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2655 void JEMALLOC_NOTHROW *
2656 JEMALLOC_ALLOC_SIZE(2)
2657 je_realloc(void *ptr, size_t arg_size) {
2659 tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL);
2660 size_t usize JEMALLOC_CC_SILENCE_INIT(0);
2661 size_t old_usize = 0;
2662 size_t size = arg_size;
2664 LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);
2666 if (unlikely(size == 0)) {
2670 if (likely(ptr != NULL)) {
2671 assert(malloc_initialized() || IS_INITIALIZER);
2672 tsd_t *tsd = tsd_fetch();
2674 check_entry_exit_locking(tsd_tsdn(tsd));
2677 hook_ralloc_args_t hook_args = {true, {(uintptr_t)ptr,
2678 (uintptr_t)arg_size, 0, 0}};
2680 alloc_ctx_t alloc_ctx;
2681 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2682 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
2683 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
2684 assert(alloc_ctx.szind != SC_NSIZES);
2685 old_usize = sz_index2size(alloc_ctx.szind);
2686 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
2687 if (config_prof && opt_prof) {
2688 usize = sz_s2u(size);
2689 if (unlikely(usize == 0
2690 || usize > SC_LARGE_MAXCLASS)) {
2693 ret = irealloc_prof(tsd, ptr, old_usize, usize,
2694 &alloc_ctx, &hook_args);
2698 usize = sz_s2u(size);
2700 ret = iralloc(tsd, ptr, old_usize, size, 0, false,
2703 tsdn = tsd_tsdn(tsd);
2705 /* realloc(NULL, size) is equivalent to malloc(size). */
2706 static_opts_t sopts;
2707 dynamic_opts_t dopts;
2709 static_opts_init(&sopts);
2710 dynamic_opts_init(&dopts);
2712 sopts.null_out_result_on_error = true;
2713 sopts.set_errno_on_error = true;
2715 "<jemalloc>: Error in realloc(): out of memory\n";
2717 dopts.result = &ret;
2718 dopts.num_items = 1;
2719 dopts.item_size = size;
2721 imalloc(&sopts, &dopts);
2723 uintptr_t args[3] = {(uintptr_t)ptr, arg_size};
2724 hook_invoke_alloc(hook_alloc_realloc, ret,
2725 (uintptr_t)ret, args);
2731 if (unlikely(ret == NULL)) {
2732 if (config_xmalloc && unlikely(opt_xmalloc)) {
2733 malloc_write("<jemalloc>: Error in realloc(): "
2739 if (config_stats && likely(ret != NULL)) {
2742 assert(usize == isalloc(tsdn, ret));
2743 tsd = tsdn_tsd(tsdn);
2744 *tsd_thread_allocatedp_get(tsd) += usize;
2745 *tsd_thread_deallocatedp_get(tsd) += old_usize;
2747 UTRACE(ptr, size, ret);
2748 check_entry_exit_locking(tsdn);
2750 LOG("core.realloc.exit", "result: %p", ret);
2756 free_default(void *ptr) {
2758 if (likely(ptr != NULL)) {
2760 * We avoid setting up tsd fully (e.g. tcache, arena binding)
2761 * based on only free() calls -- other activities trigger the
2762 * minimal to full transition. This is because free() may
2763 * happen during thread shutdown after tls deallocation: if a
2764 * thread never had any malloc activities until then, a
2765 * fully-setup tsd won't be destructed properly.
2767 tsd_t *tsd = tsd_fetch_min();
2768 check_entry_exit_locking(tsd_tsdn(tsd));
2771 if (likely(tsd_fast(tsd))) {
2772 tsd_assert_fast(tsd);
2773 /* Unconditionally get tcache ptr on fast path. */
2774 tcache = tsd_tcachep_get(tsd);
2775 ifree(tsd, ptr, tcache, false);
2777 if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
2778 tcache = tcache_get(tsd);
2782 uintptr_t args_raw[3] = {(uintptr_t)ptr};
2783 hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw);
2784 ifree(tsd, ptr, tcache, true);
2786 check_entry_exit_locking(tsd_tsdn(tsd));
2790 JEMALLOC_ALWAYS_INLINE
2791 bool free_fastpath(void *ptr, size_t size, bool size_hint) {
2792 tsd_t *tsd = tsd_get(false);
2793 if (unlikely(!tsd || !tsd_fast(tsd))) {
2797 tcache_t *tcache = tsd_tcachep_get(tsd);
2799 alloc_ctx_t alloc_ctx;
2801 * If !config_cache_oblivious, we can check PAGE alignment to
2802 * detect sampled objects. Otherwise addresses are
2803 * randomized, and we have to look it up in the rtree anyway.
2804 * See also isfree().
2806 if (!size_hint || config_cache_oblivious) {
2807 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
2808 bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), &extents_rtree,
2809 rtree_ctx, (uintptr_t)ptr,
2810 &alloc_ctx.szind, &alloc_ctx.slab);
2812 /* Note: profiled objects will have alloc_ctx.slab set */
2813 if (!res || !alloc_ctx.slab) {
2816 assert(alloc_ctx.szind != SC_NSIZES);
2819 * Check for both sizes that are too large, and for sampled objects.
2820 * Sampled objects are always page-aligned. The sampled object check
2821 * will also check for null ptr.
2823 if (size > SC_LOOKUP_MAXCLASS || (((uintptr_t)ptr & PAGE_MASK) == 0)) {
2826 alloc_ctx.szind = sz_size2index_lookup(size);
2829 if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
2833 cache_bin_t *bin = tcache_small_bin_get(tcache, alloc_ctx.szind);
2834 cache_bin_info_t *bin_info = &tcache_bin_info[alloc_ctx.szind];
2835 if (!cache_bin_dalloc_easy(bin, bin_info, ptr)) {
2840 size_t usize = sz_index2size(alloc_ctx.szind);
2841 *tsd_thread_deallocatedp_get(tsd) += usize;
2847 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
2848 je_free(void *ptr) {
2849 LOG("core.free.entry", "ptr: %p", ptr);
2851 if (!free_fastpath(ptr, 0, false)) {
2855 LOG("core.free.exit", "");
2859 * End malloc(3)-compatible functions.
2861 /******************************************************************************/
2863 * Begin non-standard override functions.
2866 #ifdef JEMALLOC_OVERRIDE_MEMALIGN
2867 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2868 void JEMALLOC_NOTHROW *
2869 JEMALLOC_ATTR(malloc)
2870 je_memalign(size_t alignment, size_t size) {
2872 static_opts_t sopts;
2873 dynamic_opts_t dopts;
2875 LOG("core.memalign.entry", "alignment: %zu, size: %zu\n", alignment,
2878 static_opts_init(&sopts);
2879 dynamic_opts_init(&dopts);
2881 sopts.min_alignment = 1;
2883 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2884 sopts.invalid_alignment_string =
2885 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2886 sopts.null_out_result_on_error = true;
2888 dopts.result = &ret;
2889 dopts.num_items = 1;
2890 dopts.item_size = size;
2891 dopts.alignment = alignment;
2893 imalloc(&sopts, &dopts);
2895 uintptr_t args[3] = {alignment, size};
2896 hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret,
2900 LOG("core.memalign.exit", "result: %p", ret);
2905 #ifdef JEMALLOC_OVERRIDE_VALLOC
2906 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
2907 void JEMALLOC_NOTHROW *
2908 JEMALLOC_ATTR(malloc)
2909 je_valloc(size_t size) {
2912 static_opts_t sopts;
2913 dynamic_opts_t dopts;
2915 LOG("core.valloc.entry", "size: %zu\n", size);
2917 static_opts_init(&sopts);
2918 dynamic_opts_init(&dopts);
2920 sopts.null_out_result_on_error = true;
2921 sopts.min_alignment = PAGE;
2923 "<jemalloc>: Error allocating aligned memory: out of memory\n";
2924 sopts.invalid_alignment_string =
2925 "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
2927 dopts.result = &ret;
2928 dopts.num_items = 1;
2929 dopts.item_size = size;
2930 dopts.alignment = PAGE;
2932 imalloc(&sopts, &dopts);
2934 uintptr_t args[3] = {size};
2935 hook_invoke_alloc(hook_alloc_valloc, ret, (uintptr_t)ret, args);
2938 LOG("core.valloc.exit", "result: %p\n", ret);
2943 #if defined(JEMALLOC_IS_MALLOC) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)
2945 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
2946 * to inconsistently reference libc's malloc(3)-compatible functions
2947 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
2949 * These definitions interpose hooks in glibc. The functions are actually
2950 * passed an extra argument for the caller return address, which will be
2953 JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
2954 JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
2955 JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
2956 # ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK
2957 JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
2963 * To enable static linking with glibc, the libc specific malloc interface must
2964 * be implemented also, so none of glibc's malloc.o functions are added to the
2967 # define ALIAS(je_fn) __attribute__((alias (#je_fn), used))
2968 /* To force macro expansion of je_ prefix before stringification. */
2969 # define PREALIAS(je_fn) ALIAS(je_fn)
2970 # ifdef JEMALLOC_OVERRIDE___LIBC_CALLOC
2971 void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc);
2973 # ifdef JEMALLOC_OVERRIDE___LIBC_FREE
2974 void __libc_free(void* ptr) PREALIAS(je_free);
2976 # ifdef JEMALLOC_OVERRIDE___LIBC_MALLOC
2977 void *__libc_malloc(size_t size) PREALIAS(je_malloc);
2979 # ifdef JEMALLOC_OVERRIDE___LIBC_MEMALIGN
2980 void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign);
2982 # ifdef JEMALLOC_OVERRIDE___LIBC_REALLOC
2983 void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc);
2985 # ifdef JEMALLOC_OVERRIDE___LIBC_VALLOC
2986 void *__libc_valloc(size_t size) PREALIAS(je_valloc);
2988 # ifdef JEMALLOC_OVERRIDE___POSIX_MEMALIGN
2989 int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign);
2997 * End non-standard override functions.
2999 /******************************************************************************/
3001 * Begin non-standard functions.
3004 #ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API
3006 #define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y
3007 #define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \
3008 JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y)
3013 } smallocx_return_t;
3015 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3016 smallocx_return_t JEMALLOC_NOTHROW
3018 * The attribute JEMALLOC_ATTR(malloc) cannot be used due to:
3019 * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488
3021 JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT)
3022 (size_t size, int flags) {
3024 * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be
3025 * used here because it makes writing beyond the `size`
3026 * of the `ptr` undefined behavior, but the objective
3027 * of this function is to allow writing beyond `size`
3028 * up to `smallocx_return_t::size`.
3030 smallocx_return_t ret;
3031 static_opts_t sopts;
3032 dynamic_opts_t dopts;
3034 LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags);
3036 static_opts_init(&sopts);
3037 dynamic_opts_init(&dopts);
3039 sopts.assert_nonempty_alloc = true;
3040 sopts.null_out_result_on_error = true;
3041 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n";
3044 dopts.result = &ret.ptr;
3045 dopts.num_items = 1;
3046 dopts.item_size = size;
3047 if (unlikely(flags != 0)) {
3048 if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) {
3049 dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
3052 dopts.zero = MALLOCX_ZERO_GET(flags);
3054 if ((flags & MALLOCX_TCACHE_MASK) != 0) {
3055 if ((flags & MALLOCX_TCACHE_MASK)
3056 == MALLOCX_TCACHE_NONE) {
3057 dopts.tcache_ind = TCACHE_IND_NONE;
3059 dopts.tcache_ind = MALLOCX_TCACHE_GET(flags);
3062 dopts.tcache_ind = TCACHE_IND_AUTOMATIC;
3065 if ((flags & MALLOCX_ARENA_MASK) != 0)
3066 dopts.arena_ind = MALLOCX_ARENA_GET(flags);
3069 imalloc(&sopts, &dopts);
3070 assert(dopts.usize == je_nallocx(size, flags));
3071 ret.size = dopts.usize;
3073 LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size);
3076 #undef JEMALLOC_SMALLOCX_CONCAT_HELPER
3077 #undef JEMALLOC_SMALLOCX_CONCAT_HELPER2
3080 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3081 void JEMALLOC_NOTHROW *
3082 JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
3083 je_mallocx(size_t size, int flags) {
3085 static_opts_t sopts;
3086 dynamic_opts_t dopts;
3088 LOG("core.mallocx.entry", "size: %zu, flags: %d", size, flags);
3090 static_opts_init(&sopts);
3091 dynamic_opts_init(&dopts);
3093 sopts.assert_nonempty_alloc = true;
3094 sopts.null_out_result_on_error = true;
3095 sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n";
3097 dopts.result = &ret;
3098 dopts.num_items = 1;
3099 dopts.item_size = size;
3100 if (unlikely(flags != 0)) {
3101 if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) {
3102 dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
3105 dopts.zero = MALLOCX_ZERO_GET(flags);
3107 if ((flags & MALLOCX_TCACHE_MASK) != 0) {
3108 if ((flags & MALLOCX_TCACHE_MASK)
3109 == MALLOCX_TCACHE_NONE) {
3110 dopts.tcache_ind = TCACHE_IND_NONE;
3112 dopts.tcache_ind = MALLOCX_TCACHE_GET(flags);
3115 dopts.tcache_ind = TCACHE_IND_AUTOMATIC;
3118 if ((flags & MALLOCX_ARENA_MASK) != 0)
3119 dopts.arena_ind = MALLOCX_ARENA_GET(flags);
3122 imalloc(&sopts, &dopts);
3124 uintptr_t args[3] = {size, flags};
3125 hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret,
3129 LOG("core.mallocx.exit", "result: %p", ret);
3134 irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize,
3135 size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
3136 prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) {
3142 if (usize <= SC_SMALL_MAXCLASS) {
3143 p = iralloct(tsdn, old_ptr, old_usize,
3144 SC_LARGE_MINCLASS, alignment, zero, tcache,
3149 arena_prof_promote(tsdn, p, usize);
3151 p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero,
3152 tcache, arena, hook_args);
3158 JEMALLOC_ALWAYS_INLINE void *
3159 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
3160 size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
3161 arena_t *arena, alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) {
3164 prof_tctx_t *old_tctx, *tctx;
3166 prof_active = prof_active_get_unlocked();
3167 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx);
3168 tctx = prof_alloc_prep(tsd, *usize, prof_active, false);
3169 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
3170 p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize,
3171 *usize, alignment, zero, tcache, arena, tctx, hook_args);
3173 p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment,
3174 zero, tcache, arena, hook_args);
3176 if (unlikely(p == NULL)) {
3177 prof_alloc_rollback(tsd, tctx, false);
3181 if (p == old_ptr && alignment != 0) {
3183 * The allocation did not move, so it is possible that the size
3184 * class is smaller than would guarantee the requested
3185 * alignment, and that the alignment constraint was
3186 * serendipitously satisfied. Additionally, old_usize may not
3187 * be the same as the current usize because of in-place large
3188 * reallocation. Therefore, query the actual value of usize.
3190 *usize = isalloc(tsd_tsdn(tsd), p);
3192 prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr,
3193 old_usize, old_tctx);
3198 JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
3199 void JEMALLOC_NOTHROW *
3200 JEMALLOC_ALLOC_SIZE(2)
3201 je_rallocx(void *ptr, size_t size, int flags) {
3206 size_t alignment = MALLOCX_ALIGN_GET(flags);
3207 bool zero = flags & MALLOCX_ZERO;
3211 LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
3215 assert(ptr != NULL);
3217 assert(malloc_initialized() || IS_INITIALIZER);
3219 check_entry_exit_locking(tsd_tsdn(tsd));
3221 if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
3222 unsigned arena_ind = MALLOCX_ARENA_GET(flags);
3223 arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
3224 if (unlikely(arena == NULL)) {
3231 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
3232 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
3235 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
3238 tcache = tcache_get(tsd);
3241 alloc_ctx_t alloc_ctx;
3242 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
3243 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
3244 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
3245 assert(alloc_ctx.szind != SC_NSIZES);
3246 old_usize = sz_index2size(alloc_ctx.szind);
3247 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
3249 hook_ralloc_args_t hook_args = {false, {(uintptr_t)ptr, size, flags,
3251 if (config_prof && opt_prof) {
3252 usize = (alignment == 0) ?
3253 sz_s2u(size) : sz_sa2u(size, alignment);
3254 if (unlikely(usize == 0
3255 || usize > SC_LARGE_MAXCLASS)) {
3258 p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
3259 zero, tcache, arena, &alloc_ctx, &hook_args);
3260 if (unlikely(p == NULL)) {
3264 p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment,
3265 zero, tcache, arena, &hook_args);
3266 if (unlikely(p == NULL)) {
3270 usize = isalloc(tsd_tsdn(tsd), p);
3273 assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
3276 *tsd_thread_allocatedp_get(tsd) += usize;
3277 *tsd_thread_deallocatedp_get(tsd) += old_usize;
3279 UTRACE(ptr, size, p);
3280 check_entry_exit_locking(tsd_tsdn(tsd));
3282 LOG("core.rallocx.exit", "result: %p", p);
3285 if (config_xmalloc && unlikely(opt_xmalloc)) {
3286 malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
3289 UTRACE(ptr, size, 0);
3290 check_entry_exit_locking(tsd_tsdn(tsd));
3292 LOG("core.rallocx.exit", "result: %p", NULL);
3296 JEMALLOC_ALWAYS_INLINE size_t
3297 ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
3298 size_t extra, size_t alignment, bool zero) {
3301 if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero,
3310 ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
3311 size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) {
3317 usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
3323 JEMALLOC_ALWAYS_INLINE size_t
3324 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
3325 size_t extra, size_t alignment, bool zero, alloc_ctx_t *alloc_ctx) {
3326 size_t usize_max, usize;
3328 prof_tctx_t *old_tctx, *tctx;
3330 prof_active = prof_active_get_unlocked();
3331 old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr, alloc_ctx);
3333 * usize isn't knowable before ixalloc() returns when extra is non-zero.
3334 * Therefore, compute its maximum possible value and use that in
3335 * prof_alloc_prep() to decide whether to capture a backtrace.
3336 * prof_realloc() will use the actual usize to decide whether to sample.
3338 if (alignment == 0) {
3339 usize_max = sz_s2u(size+extra);
3340 assert(usize_max > 0
3341 && usize_max <= SC_LARGE_MAXCLASS);
3343 usize_max = sz_sa2u(size+extra, alignment);
3344 if (unlikely(usize_max == 0
3345 || usize_max > SC_LARGE_MAXCLASS)) {
3347 * usize_max is out of range, and chances are that
3348 * allocation will fail, but use the maximum possible
3349 * value and carry on with prof_alloc_prep(), just in
3350 * case allocation succeeds.
3352 usize_max = SC_LARGE_MAXCLASS;
3355 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
3357 if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
3358 usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
3359 size, extra, alignment, zero, tctx);
3361 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
3362 extra, alignment, zero);
3364 if (usize == old_usize) {
3365 prof_alloc_rollback(tsd, tctx, false);
3368 prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
3374 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3375 je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
3377 size_t usize, old_usize;
3378 size_t alignment = MALLOCX_ALIGN_GET(flags);
3379 bool zero = flags & MALLOCX_ZERO;
3381 LOG("core.xallocx.entry", "ptr: %p, size: %zu, extra: %zu, "
3382 "flags: %d", ptr, size, extra, flags);
3384 assert(ptr != NULL);
3386 assert(SIZE_T_MAX - size >= extra);
3387 assert(malloc_initialized() || IS_INITIALIZER);
3389 check_entry_exit_locking(tsd_tsdn(tsd));
3391 alloc_ctx_t alloc_ctx;
3392 rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
3393 rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
3394 (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
3395 assert(alloc_ctx.szind != SC_NSIZES);
3396 old_usize = sz_index2size(alloc_ctx.szind);
3397 assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
3399 * The API explicitly absolves itself of protecting against (size +
3400 * extra) numerical overflow, but we may need to clamp extra to avoid
3401 * exceeding SC_LARGE_MAXCLASS.
3403 * Ordinarily, size limit checking is handled deeper down, but here we
3404 * have to check as part of (size + extra) clamping, since we need the
3405 * clamped value in the above helper functions.
3407 if (unlikely(size > SC_LARGE_MAXCLASS)) {
3409 goto label_not_resized;
3411 if (unlikely(SC_LARGE_MAXCLASS - size < extra)) {
3412 extra = SC_LARGE_MAXCLASS - size;
3415 if (config_prof && opt_prof) {
3416 usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
3417 alignment, zero, &alloc_ctx);
3419 usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
3420 extra, alignment, zero);
3422 if (unlikely(usize == old_usize)) {
3423 goto label_not_resized;
3427 *tsd_thread_allocatedp_get(tsd) += usize;
3428 *tsd_thread_deallocatedp_get(tsd) += old_usize;
3431 if (unlikely(!tsd_fast(tsd))) {
3432 uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags};
3433 hook_invoke_expand(hook_expand_xallocx, ptr, old_usize,
3434 usize, (uintptr_t)usize, args);
3437 UTRACE(ptr, size, ptr);
3438 check_entry_exit_locking(tsd_tsdn(tsd));
3440 LOG("core.xallocx.exit", "result: %zu", usize);
3444 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3446 je_sallocx(const void *ptr, int flags) {
3450 LOG("core.sallocx.entry", "ptr: %p, flags: %d", ptr, flags);
3452 assert(malloc_initialized() || IS_INITIALIZER);
3453 assert(ptr != NULL);
3455 tsdn = tsdn_fetch();
3456 check_entry_exit_locking(tsdn);
3458 if (config_debug || force_ivsalloc) {
3459 usize = ivsalloc(tsdn, ptr);
3460 assert(force_ivsalloc || usize != 0);
3462 usize = isalloc(tsdn, ptr);
3465 check_entry_exit_locking(tsdn);
3467 LOG("core.sallocx.exit", "result: %zu", usize);
3471 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
3472 je_dallocx(void *ptr, int flags) {
3473 LOG("core.dallocx.entry", "ptr: %p, flags: %d", ptr, flags);
3475 assert(ptr != NULL);
3476 assert(malloc_initialized() || IS_INITIALIZER);
3478 tsd_t *tsd = tsd_fetch();
3479 bool fast = tsd_fast(tsd);
3480 check_entry_exit_locking(tsd_tsdn(tsd));
3483 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
3484 /* Not allowed to be reentrant and specify a custom tcache. */
3485 assert(tsd_reentrancy_level_get(tsd) == 0);
3486 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
3489 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
3493 tcache = tsd_tcachep_get(tsd);
3494 assert(tcache == tcache_get(tsd));
3496 if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
3497 tcache = tcache_get(tsd);
3506 tsd_assert_fast(tsd);
3507 ifree(tsd, ptr, tcache, false);
3509 uintptr_t args_raw[3] = {(uintptr_t)ptr, flags};
3510 hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw);
3511 ifree(tsd, ptr, tcache, true);
3513 check_entry_exit_locking(tsd_tsdn(tsd));
3515 LOG("core.dallocx.exit", "");
3518 JEMALLOC_ALWAYS_INLINE size_t
3519 inallocx(tsdn_t *tsdn, size_t size, int flags) {
3520 check_entry_exit_locking(tsdn);
3523 if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) {
3524 usize = sz_s2u(size);
3526 usize = sz_sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
3528 check_entry_exit_locking(tsdn);
3532 JEMALLOC_NOINLINE void
3533 sdallocx_default(void *ptr, size_t size, int flags) {
3534 assert(ptr != NULL);
3535 assert(malloc_initialized() || IS_INITIALIZER);
3537 tsd_t *tsd = tsd_fetch();
3538 bool fast = tsd_fast(tsd);
3539 size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
3540 assert(usize == isalloc(tsd_tsdn(tsd), ptr));
3541 check_entry_exit_locking(tsd_tsdn(tsd));
3544 if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
3545 /* Not allowed to be reentrant and specify a custom tcache. */
3546 assert(tsd_reentrancy_level_get(tsd) == 0);
3547 if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
3550 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
3554 tcache = tsd_tcachep_get(tsd);
3555 assert(tcache == tcache_get(tsd));
3557 if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
3558 tcache = tcache_get(tsd);
3567 tsd_assert_fast(tsd);
3568 isfree(tsd, ptr, usize, tcache, false);
3570 uintptr_t args_raw[3] = {(uintptr_t)ptr, size, flags};
3571 hook_invoke_dalloc(hook_dalloc_sdallocx, ptr, args_raw);
3572 isfree(tsd, ptr, usize, tcache, true);
3574 check_entry_exit_locking(tsd_tsdn(tsd));
3578 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
3579 je_sdallocx(void *ptr, size_t size, int flags) {
3580 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
3583 if (flags !=0 || !free_fastpath(ptr, size, true)) {
3584 sdallocx_default(ptr, size, flags);
3587 LOG("core.sdallocx.exit", "");
3590 void JEMALLOC_NOTHROW
3591 je_sdallocx_noflags(void *ptr, size_t size) {
3592 LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: 0", ptr,
3595 if (!free_fastpath(ptr, size, true)) {
3596 sdallocx_default(ptr, size, 0);
3599 LOG("core.sdallocx.exit", "");
3602 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3604 je_nallocx(size_t size, int flags) {
3610 if (unlikely(malloc_init())) {
3611 LOG("core.nallocx.exit", "result: %zu", ZU(0));
3615 tsdn = tsdn_fetch();
3616 check_entry_exit_locking(tsdn);
3618 usize = inallocx(tsdn, size, flags);
3619 if (unlikely(usize > SC_LARGE_MAXCLASS)) {
3620 LOG("core.nallocx.exit", "result: %zu", ZU(0));
3624 check_entry_exit_locking(tsdn);
3625 LOG("core.nallocx.exit", "result: %zu", usize);
3629 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
3630 je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
3635 LOG("core.mallctl.entry", "name: %s", name);
3637 if (unlikely(malloc_init())) {
3638 LOG("core.mallctl.exit", "result: %d", EAGAIN);
3643 check_entry_exit_locking(tsd_tsdn(tsd));
3644 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
3645 check_entry_exit_locking(tsd_tsdn(tsd));
3647 LOG("core.mallctl.exit", "result: %d", ret);
3651 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
3652 je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) {
3655 LOG("core.mallctlnametomib.entry", "name: %s", name);
3657 if (unlikely(malloc_init())) {
3658 LOG("core.mallctlnametomib.exit", "result: %d", EAGAIN);
3662 tsd_t *tsd = tsd_fetch();
3663 check_entry_exit_locking(tsd_tsdn(tsd));
3664 ret = ctl_nametomib(tsd, name, mibp, miblenp);
3665 check_entry_exit_locking(tsd_tsdn(tsd));
3667 LOG("core.mallctlnametomib.exit", "result: %d", ret);
3671 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
3672 je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
3673 void *newp, size_t newlen) {
3677 LOG("core.mallctlbymib.entry", "");
3679 if (unlikely(malloc_init())) {
3680 LOG("core.mallctlbymib.exit", "result: %d", EAGAIN);
3685 check_entry_exit_locking(tsd_tsdn(tsd));
3686 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
3687 check_entry_exit_locking(tsd_tsdn(tsd));
3688 LOG("core.mallctlbymib.exit", "result: %d", ret);
3692 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
3693 je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
3697 LOG("core.malloc_stats_print.entry", "");
3699 tsdn = tsdn_fetch();
3700 check_entry_exit_locking(tsdn);
3701 stats_print(write_cb, cbopaque, opts);
3702 check_entry_exit_locking(tsdn);
3703 LOG("core.malloc_stats_print.exit", "");
3706 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
3707 je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
3711 LOG("core.malloc_usable_size.entry", "ptr: %p", ptr);
3713 assert(malloc_initialized() || IS_INITIALIZER);
3715 tsdn = tsdn_fetch();
3716 check_entry_exit_locking(tsdn);
3718 if (unlikely(ptr == NULL)) {
3721 if (config_debug || force_ivsalloc) {
3722 ret = ivsalloc(tsdn, ptr);
3723 assert(force_ivsalloc || ret != 0);
3725 ret = isalloc(tsdn, ptr);
3729 check_entry_exit_locking(tsdn);
3730 LOG("core.malloc_usable_size.exit", "result: %zu", ret);
3735 * End non-standard functions.
3737 /******************************************************************************/
3739 * Begin compatibility functions.
3742 #define ALLOCM_LG_ALIGN(la) (la)
3743 #define ALLOCM_ALIGN(a) (ffsl(a)-1)
3744 #define ALLOCM_ZERO ((int)0x40)
3745 #define ALLOCM_NO_MOVE ((int)0x80)
3747 #define ALLOCM_SUCCESS 0
3748 #define ALLOCM_ERR_OOM 1
3749 #define ALLOCM_ERR_NOT_MOVED 2
3752 je_allocm(void **ptr, size_t *rsize, size_t size, int flags) {
3753 assert(ptr != NULL);
3755 void *p = je_mallocx(size, flags);
3757 return (ALLOCM_ERR_OOM);
3759 if (rsize != NULL) {
3760 *rsize = isalloc(tsdn_fetch(), p);
3763 return ALLOCM_SUCCESS;
3767 je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) {
3768 assert(ptr != NULL);
3769 assert(*ptr != NULL);
3771 assert(SIZE_T_MAX - size >= extra);
3774 bool no_move = flags & ALLOCM_NO_MOVE;
3777 size_t usize = je_xallocx(*ptr, size, extra, flags);
3778 ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED;
3779 if (rsize != NULL) {
3783 void *p = je_rallocx(*ptr, size+extra, flags);
3786 ret = ALLOCM_SUCCESS;
3788 ret = ALLOCM_ERR_OOM;
3790 if (rsize != NULL) {
3791 *rsize = isalloc(tsdn_fetch(), *ptr);
3798 je_sallocm(const void *ptr, size_t *rsize, int flags) {
3799 assert(rsize != NULL);
3800 *rsize = je_sallocx(ptr, flags);
3801 return ALLOCM_SUCCESS;
3805 je_dallocm(void *ptr, int flags) {
3806 je_dallocx(ptr, flags);
3807 return ALLOCM_SUCCESS;
3811 je_nallocm(size_t *rsize, size_t size, int flags) {
3812 size_t usize = je_nallocx(size, flags);
3814 return ALLOCM_ERR_OOM;
3816 if (rsize != NULL) {
3819 return ALLOCM_SUCCESS;
3822 #undef ALLOCM_LG_ALIGN
3825 #undef ALLOCM_NO_MOVE
3827 #undef ALLOCM_SUCCESS
3828 #undef ALLOCM_ERR_OOM
3829 #undef ALLOCM_ERR_NOT_MOVED
3832 * End compatibility functions.
3834 /******************************************************************************/
3836 * The following functions are used by threading libraries for protection of
3837 * malloc during fork().
3841 * If an application creates a thread before doing any allocation in the main
3842 * thread, then calls fork(2) in the main thread followed by memory allocation
3843 * in the child process, a race can occur that results in deadlock within the
3844 * child: the main thread may have forked while the created thread had
3845 * partially initialized the allocator. Ordinarily jemalloc prevents
3846 * fork/malloc races via the following functions it registers during
3847 * initialization using pthread_atfork(), but of course that does no good if
3848 * the allocator isn't fully initialized at fork time. The following library
3849 * constructor is a partial solution to this problem. It may still be possible
3850 * to trigger the deadlock described above, but doing so would involve forking
3851 * via a library constructor that runs before jemalloc's runs.
3853 #ifndef JEMALLOC_JET
3854 JEMALLOC_ATTR(constructor)
3856 jemalloc_constructor(void) {
3861 #ifndef JEMALLOC_MUTEX_INIT_CB
3863 jemalloc_prefork(void)
3865 JEMALLOC_EXPORT void
3866 _malloc_prefork(void)
3870 unsigned i, j, narenas;
3873 #ifdef JEMALLOC_MUTEX_INIT_CB
3874 if (!malloc_initialized()) {
3878 assert(malloc_initialized());
3882 narenas = narenas_total_get();
3884 witness_prefork(tsd_witness_tsdp_get(tsd));
3885 /* Acquire all mutexes in a safe order. */
3886 ctl_prefork(tsd_tsdn(tsd));
3887 tcache_prefork(tsd_tsdn(tsd));
3888 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
3889 if (have_background_thread) {
3890 background_thread_prefork0(tsd_tsdn(tsd));
3892 prof_prefork0(tsd_tsdn(tsd));
3893 if (have_background_thread) {
3894 background_thread_prefork1(tsd_tsdn(tsd));
3896 /* Break arena prefork into stages to preserve lock order. */
3897 for (i = 0; i < 8; i++) {
3898 for (j = 0; j < narenas; j++) {
3899 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) !=
3903 arena_prefork0(tsd_tsdn(tsd), arena);
3906 arena_prefork1(tsd_tsdn(tsd), arena);
3909 arena_prefork2(tsd_tsdn(tsd), arena);
3912 arena_prefork3(tsd_tsdn(tsd), arena);
3915 arena_prefork4(tsd_tsdn(tsd), arena);
3918 arena_prefork5(tsd_tsdn(tsd), arena);
3921 arena_prefork6(tsd_tsdn(tsd), arena);
3924 arena_prefork7(tsd_tsdn(tsd), arena);
3926 default: not_reached();
3931 prof_prefork1(tsd_tsdn(tsd));
3935 #ifndef JEMALLOC_MUTEX_INIT_CB
3937 jemalloc_postfork_parent(void)
3939 JEMALLOC_EXPORT void
3940 _malloc_postfork(void)
3944 unsigned i, narenas;
3946 #ifdef JEMALLOC_MUTEX_INIT_CB
3947 if (!malloc_initialized()) {
3951 assert(malloc_initialized());
3955 tsd_postfork_parent(tsd);
3957 witness_postfork_parent(tsd_witness_tsdp_get(tsd));
3958 /* Release all mutexes, now that fork() has completed. */
3959 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
3962 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
3963 arena_postfork_parent(tsd_tsdn(tsd), arena);
3966 prof_postfork_parent(tsd_tsdn(tsd));
3967 if (have_background_thread) {
3968 background_thread_postfork_parent(tsd_tsdn(tsd));
3970 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
3971 tcache_postfork_parent(tsd_tsdn(tsd));
3972 ctl_postfork_parent(tsd_tsdn(tsd));
3976 jemalloc_postfork_child(void) {
3978 unsigned i, narenas;
3980 assert(malloc_initialized());
3984 tsd_postfork_child(tsd);
3986 witness_postfork_child(tsd_witness_tsdp_get(tsd));
3987 /* Release all mutexes, now that fork() has completed. */
3988 for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
3991 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
3992 arena_postfork_child(tsd_tsdn(tsd), arena);
3995 prof_postfork_child(tsd_tsdn(tsd));
3996 if (have_background_thread) {
3997 background_thread_postfork_child(tsd_tsdn(tsd));
3999 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
4000 tcache_postfork_child(tsd_tsdn(tsd));
4001 ctl_postfork_child(tsd_tsdn(tsd));
4005 _malloc_first_thread(void)
4008 (void)malloc_mutex_first_thread();
4011 /******************************************************************************/