1 #define JEMALLOC_PROF_C_
2 #include "jemalloc/internal/jemalloc_preamble.h"
3 #include "jemalloc/internal/jemalloc_internal_includes.h"
5 #include "jemalloc/internal/assert.h"
6 #include "jemalloc/internal/ckh.h"
7 #include "jemalloc/internal/hash.h"
8 #include "jemalloc/internal/malloc_io.h"
9 #include "jemalloc/internal/mutex.h"
11 /******************************************************************************/
13 #ifdef JEMALLOC_PROF_LIBUNWIND
14 #define UNW_LOCAL_ONLY
15 #include <libunwind.h>
18 #ifdef JEMALLOC_PROF_LIBGCC
20 * We have a circular dependency -- jemalloc_internal.h tells us if we should
21 * use libgcc's unwinding functionality, but after we've included that, we've
22 * already hooked _Unwind_Backtrace. We'll temporarily disable hooking.
24 #undef _Unwind_Backtrace
26 #define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook)
29 /******************************************************************************/
32 bool opt_prof = false;
33 bool opt_prof_active = true;
34 bool opt_prof_thread_active_init = true;
35 size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
36 ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
37 bool opt_prof_gdump = false;
38 bool opt_prof_final = false;
39 bool opt_prof_leak = false;
40 bool opt_prof_accum = false;
42 /* Minimize memory bloat for non-prof builds. */
49 * Initialized as opt_prof_active, and accessed via
50 * prof_active_[gs]et{_unlocked,}().
53 static malloc_mutex_t prof_active_mtx;
56 * Initialized as opt_prof_thread_active_init, and accessed via
57 * prof_thread_active_init_[gs]et().
59 static bool prof_thread_active_init;
60 static malloc_mutex_t prof_thread_active_init_mtx;
63 * Initialized as opt_prof_gdump, and accessed via
64 * prof_gdump_[gs]et{_unlocked,}().
67 static malloc_mutex_t prof_gdump_mtx;
69 uint64_t prof_interval = 0;
71 size_t lg_prof_sample;
74 * Table of mutexes that are shared among gctx's. These are leaf locks, so
75 * there is no problem with using them for more than one gctx at the same time.
76 * The primary motivation for this sharing though is that gctx's are ephemeral,
77 * and destroying mutexes causes complications for systems that allocate when
78 * creating/destroying mutexes.
80 static malloc_mutex_t *gctx_locks;
81 static atomic_u_t cum_gctxs; /* Atomic counter. */
84 * Table of mutexes that are shared among tdata's. No operations require
85 * holding multiple tdata locks, so there is no problem with using them for more
86 * than one tdata at the same time, even though a gctx lock may be acquired
87 * while holding a tdata lock.
89 static malloc_mutex_t *tdata_locks;
92 * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data
93 * structure that knows about all backtraces currently captured.
96 /* Non static to enable profiling. */
97 malloc_mutex_t bt2gctx_mtx;
100 * Tree of all extant prof_tdata_t structures, regardless of state,
101 * {attached,detached,expired}.
103 static prof_tdata_tree_t tdatas;
104 static malloc_mutex_t tdatas_mtx;
106 static uint64_t next_thr_uid;
107 static malloc_mutex_t next_thr_uid_mtx;
109 static malloc_mutex_t prof_dump_seq_mtx;
110 static uint64_t prof_dump_seq;
111 static uint64_t prof_dump_iseq;
112 static uint64_t prof_dump_mseq;
113 static uint64_t prof_dump_useq;
116 * This buffer is rather large for stack allocation, so use a single buffer for
119 static malloc_mutex_t prof_dump_mtx;
120 static char prof_dump_buf[
121 /* Minimize memory bloat for non-prof builds. */
128 static size_t prof_dump_buf_end;
129 static int prof_dump_fd;
131 /* Do not dump any profiles until bootstrapping is complete. */
132 static bool prof_booted = false;
134 /******************************************************************************/
136 * Function prototypes for static functions that are referenced prior to
140 static bool prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx);
141 static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
142 static bool prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
143 bool even_if_attached);
144 static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
145 bool even_if_attached);
146 static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name);
148 /******************************************************************************/
149 /* Red-black trees. */
152 prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) {
153 uint64_t a_thr_uid = a->thr_uid;
154 uint64_t b_thr_uid = b->thr_uid;
155 int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
157 uint64_t a_thr_discrim = a->thr_discrim;
158 uint64_t b_thr_discrim = b->thr_discrim;
159 ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
162 uint64_t a_tctx_uid = a->tctx_uid;
163 uint64_t b_tctx_uid = b->tctx_uid;
164 ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
171 rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
172 tctx_link, prof_tctx_comp)
175 prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) {
176 unsigned a_len = a->bt.len;
177 unsigned b_len = b->bt.len;
178 unsigned comp_len = (a_len < b_len) ? a_len : b_len;
179 int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
181 ret = (a_len > b_len) - (a_len < b_len);
186 rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
190 prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) {
192 uint64_t a_uid = a->thr_uid;
193 uint64_t b_uid = b->thr_uid;
195 ret = ((a_uid > b_uid) - (a_uid < b_uid));
197 uint64_t a_discrim = a->thr_discrim;
198 uint64_t b_discrim = b->thr_discrim;
200 ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
205 rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
208 /******************************************************************************/
211 prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) {
214 cassert(config_prof);
218 * Compute a new sample threshold. This isn't very important in
219 * practice, because this function is rarely executed, so the
220 * potential for sample bias is minimal except in contrived
223 tdata = prof_tdata_get(tsd, true);
225 prof_sample_threshold_update(tdata);
229 if ((uintptr_t)tctx > (uintptr_t)1U) {
230 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
231 tctx->prepared = false;
232 if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) {
233 prof_tctx_destroy(tsd, tctx);
235 malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
241 prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
243 prof_tctx_set(tsdn, ptr, usize, NULL, tctx);
245 malloc_mutex_lock(tsdn, tctx->tdata->lock);
246 tctx->cnts.curobjs++;
247 tctx->cnts.curbytes += usize;
248 if (opt_prof_accum) {
249 tctx->cnts.accumobjs++;
250 tctx->cnts.accumbytes += usize;
252 tctx->prepared = false;
253 malloc_mutex_unlock(tsdn, tctx->tdata->lock);
257 prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) {
258 malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
259 assert(tctx->cnts.curobjs > 0);
260 assert(tctx->cnts.curbytes >= usize);
261 tctx->cnts.curobjs--;
262 tctx->cnts.curbytes -= usize;
264 if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) {
265 prof_tctx_destroy(tsd, tctx);
267 malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
272 bt_init(prof_bt_t *bt, void **vec) {
273 cassert(config_prof);
280 prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {
281 cassert(config_prof);
282 assert(tdata == prof_tdata_get(tsd, false));
289 malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
293 prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {
294 cassert(config_prof);
295 assert(tdata == prof_tdata_get(tsd, false));
297 malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
304 idump = tdata->enq_idump;
305 tdata->enq_idump = false;
306 gdump = tdata->enq_gdump;
307 tdata->enq_gdump = false;
310 prof_idump(tsd_tsdn(tsd));
313 prof_gdump(tsd_tsdn(tsd));
318 #ifdef JEMALLOC_PROF_LIBUNWIND
320 prof_backtrace(prof_bt_t *bt) {
323 cassert(config_prof);
324 assert(bt->len == 0);
325 assert(bt->vec != NULL);
327 nframes = unw_backtrace(bt->vec, PROF_BT_MAX);
333 #elif (defined(JEMALLOC_PROF_LIBGCC))
334 static _Unwind_Reason_Code
335 prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) {
336 cassert(config_prof);
338 return _URC_NO_REASON;
341 static _Unwind_Reason_Code
342 prof_unwind_callback(struct _Unwind_Context *context, void *arg) {
343 prof_unwind_data_t *data = (prof_unwind_data_t *)arg;
346 cassert(config_prof);
348 ip = (void *)_Unwind_GetIP(context);
350 return _URC_END_OF_STACK;
352 data->bt->vec[data->bt->len] = ip;
354 if (data->bt->len == data->max) {
355 return _URC_END_OF_STACK;
358 return _URC_NO_REASON;
362 prof_backtrace(prof_bt_t *bt) {
363 prof_unwind_data_t data = {bt, PROF_BT_MAX};
365 cassert(config_prof);
367 _Unwind_Backtrace(prof_unwind_callback, &data);
369 #elif (defined(JEMALLOC_PROF_GCC))
371 prof_backtrace(prof_bt_t *bt) {
372 #define BT_FRAME(i) \
373 if ((i) < PROF_BT_MAX) { \
375 if (__builtin_frame_address(i) == 0) { \
378 p = __builtin_return_address(i); \
388 cassert(config_prof);
534 prof_backtrace(prof_bt_t *bt) {
535 cassert(config_prof);
540 static malloc_mutex_t *
541 prof_gctx_mutex_choose(void) {
542 unsigned ngctxs = atomic_fetch_add_u(&cum_gctxs, 1, ATOMIC_RELAXED);
544 return &gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS];
547 static malloc_mutex_t *
548 prof_tdata_mutex_choose(uint64_t thr_uid) {
549 return &tdata_locks[thr_uid % PROF_NTDATA_LOCKS];
553 prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) {
555 * Create a single allocation that has space for vec of length bt->len.
557 size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *));
558 prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size,
559 sz_size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true),
564 gctx->lock = prof_gctx_mutex_choose();
566 * Set nlimbo to 1, in order to avoid a race condition with
567 * prof_tctx_destroy()/prof_gctx_try_destroy().
570 tctx_tree_new(&gctx->tctxs);
572 memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
573 gctx->bt.vec = gctx->vec;
574 gctx->bt.len = bt->len;
579 prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
580 prof_tdata_t *tdata) {
581 cassert(config_prof);
584 * Check that gctx is still unused by any thread cache before destroying
585 * it. prof_lookup() increments gctx->nlimbo in order to avoid a race
586 * condition with this function, as does prof_tctx_destroy() in order to
587 * avoid a race between the main body of prof_tctx_destroy() and entry
588 * into this function.
590 prof_enter(tsd, tdata_self);
591 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
592 assert(gctx->nlimbo != 0);
593 if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
594 /* Remove gctx from bt2gctx. */
595 if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) {
598 prof_leave(tsd, tdata_self);
600 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
601 idalloctm(tsd_tsdn(tsd), gctx, NULL, NULL, true, true);
604 * Compensate for increment in prof_tctx_destroy() or
608 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
609 prof_leave(tsd, tdata_self);
614 prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx) {
615 malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
617 if (opt_prof_accum) {
620 if (tctx->cnts.curobjs != 0) {
623 if (tctx->prepared) {
630 prof_gctx_should_destroy(prof_gctx_t *gctx) {
631 if (opt_prof_accum) {
634 if (!tctx_tree_empty(&gctx->tctxs)) {
637 if (gctx->nlimbo != 0) {
644 prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
645 prof_tdata_t *tdata = tctx->tdata;
646 prof_gctx_t *gctx = tctx->gctx;
647 bool destroy_tdata, destroy_tctx, destroy_gctx;
649 malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
651 assert(tctx->cnts.curobjs == 0);
652 assert(tctx->cnts.curbytes == 0);
653 assert(!opt_prof_accum);
654 assert(tctx->cnts.accumobjs == 0);
655 assert(tctx->cnts.accumbytes == 0);
657 ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
658 destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false);
659 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
661 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
662 switch (tctx->state) {
663 case prof_tctx_state_nominal:
664 tctx_tree_remove(&gctx->tctxs, tctx);
666 if (prof_gctx_should_destroy(gctx)) {
668 * Increment gctx->nlimbo in order to keep another
669 * thread from winning the race to destroy gctx while
670 * this one has gctx->lock dropped. Without this, it
671 * would be possible for another thread to:
673 * 1) Sample an allocation associated with gctx.
674 * 2) Deallocate the sampled object.
675 * 3) Successfully prof_gctx_try_destroy(gctx).
677 * The result would be that gctx no longer exists by the
678 * time this thread accesses it in
679 * prof_gctx_try_destroy().
684 destroy_gctx = false;
687 case prof_tctx_state_dumping:
689 * A dumping thread needs tctx to remain valid until dumping
690 * has finished. Change state such that the dumping thread will
691 * complete destruction during a late dump iteration phase.
693 tctx->state = prof_tctx_state_purgatory;
694 destroy_tctx = false;
695 destroy_gctx = false;
699 destroy_tctx = false;
700 destroy_gctx = false;
702 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
704 prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx,
708 malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock);
711 prof_tdata_destroy(tsd, tdata, false);
715 idalloctm(tsd_tsdn(tsd), tctx, NULL, NULL, true, true);
720 prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
721 void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) {
732 prof_enter(tsd, tdata);
733 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
734 /* bt has never been seen before. Insert it. */
735 prof_leave(tsd, tdata);
736 tgctx.p = prof_gctx_create(tsd_tsdn(tsd), bt);
737 if (tgctx.v == NULL) {
740 prof_enter(tsd, tdata);
741 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
743 btkey.p = &gctx.p->bt;
744 if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
746 prof_leave(tsd, tdata);
747 idalloctm(tsd_tsdn(tsd), gctx.v, NULL, NULL,
762 * Increment nlimbo, in order to avoid a race condition with
763 * prof_tctx_destroy()/prof_gctx_try_destroy().
765 malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock);
767 malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock);
770 if (tgctx.v != NULL) {
771 /* Lost race to insert. */
772 idalloctm(tsd_tsdn(tsd), tgctx.v, NULL, NULL, true,
776 prof_leave(tsd, tdata);
780 *p_new_gctx = new_gctx;
785 prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
793 cassert(config_prof);
795 tdata = prof_tdata_get(tsd, false);
800 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
801 not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
802 if (!not_found) { /* Note double negative! */
803 ret.p->prepared = true;
805 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
809 bool new_gctx, error;
812 * This thread's cache lacks bt. Look for it in the global
815 if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
820 /* Link a prof_tctx_t into gctx for this thread. */
821 ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t),
822 sz_size2index(sizeof(prof_tctx_t)), false, NULL, true,
823 arena_ichoose(tsd, NULL), true);
826 prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
830 ret.p->tdata = tdata;
831 ret.p->thr_uid = tdata->thr_uid;
832 ret.p->thr_discrim = tdata->thr_discrim;
833 memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
835 ret.p->tctx_uid = tdata->tctx_uid_next++;
836 ret.p->prepared = true;
837 ret.p->state = prof_tctx_state_initializing;
838 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
839 error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
840 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
843 prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
845 idalloctm(tsd_tsdn(tsd), ret.v, NULL, NULL, true, true);
848 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
849 ret.p->state = prof_tctx_state_nominal;
850 tctx_tree_insert(&gctx->tctxs, ret.p);
852 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
859 * The bodies of this function and prof_leakcheck() are compiled out unless heap
860 * profiling is enabled, so that it is possible to compile jemalloc with
861 * floating point support completely disabled. Avoiding floating point code is
862 * important on memory-constrained systems, but it also enables a workaround for
863 * versions of glibc that don't properly save/restore floating point registers
864 * during dynamic lazy symbol loading (which internally calls into whatever
865 * malloc implementation happens to be integrated into the application). Note
866 * that some compilers (e.g. gcc 4.8) may use floating point registers for fast
867 * memory moves, so jemalloc must be compiled with such optimizations disabled
869 * -mno-sse) in order for the workaround to be complete.
872 prof_sample_threshold_update(prof_tdata_t *tdata) {
881 if (lg_prof_sample == 0) {
882 tdata->bytes_until_sample = 0;
887 * Compute sample interval as a geometrically distributed random
888 * variable with mean (2^lg_prof_sample).
892 * tdata->bytes_until_sample = | -------- |, where p = ---------------
893 * | log(1-p) | lg_prof_sample
896 * For more information on the math, see:
898 * Non-Uniform Random Variate Generation
900 * Springer-Verlag, New York, 1986
902 * (http://luc.devroye.org/rnbookindex.html)
904 r = prng_lg_range_u64(&tdata->prng_state, 53);
905 u = (double)r * (1.0/9007199254740992.0L);
906 tdata->bytes_until_sample = (uint64_t)(log(u) /
907 log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
913 static prof_tdata_t *
914 prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
916 size_t *tdata_count = (size_t *)arg;
924 prof_tdata_count(void) {
925 size_t tdata_count = 0;
929 malloc_mutex_lock(tsdn, &tdatas_mtx);
930 tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
931 (void *)&tdata_count);
932 malloc_mutex_unlock(tsdn, &tdatas_mtx);
938 prof_bt_count(void) {
944 tdata = prof_tdata_get(tsd, false);
949 malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
950 bt_count = ckh_count(&bt2gctx);
951 malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
958 prof_dump_open_impl(bool propagate_err, const char *filename) {
961 fd = creat(filename, 0644);
962 if (fd == -1 && !propagate_err) {
963 malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n",
972 prof_dump_open_t *JET_MUTABLE prof_dump_open = prof_dump_open_impl;
975 prof_dump_flush(bool propagate_err) {
979 cassert(config_prof);
981 err = malloc_write_fd(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
983 if (!propagate_err) {
984 malloc_write("<jemalloc>: write() failed during heap "
992 prof_dump_buf_end = 0;
998 prof_dump_close(bool propagate_err) {
1001 assert(prof_dump_fd != -1);
1002 ret = prof_dump_flush(propagate_err);
1003 close(prof_dump_fd);
1010 prof_dump_write(bool propagate_err, const char *s) {
1013 cassert(config_prof);
1018 /* Flush the buffer if it is full. */
1019 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
1020 if (prof_dump_flush(propagate_err) && propagate_err) {
1025 if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) {
1026 /* Finish writing. */
1029 /* Write as much of s as will fit. */
1030 n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
1032 memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
1033 prof_dump_buf_end += n;
1040 JEMALLOC_FORMAT_PRINTF(2, 3)
1042 prof_dump_printf(bool propagate_err, const char *format, ...) {
1045 char buf[PROF_PRINTF_BUFSIZE];
1047 va_start(ap, format);
1048 malloc_vsnprintf(buf, sizeof(buf), format, ap);
1050 ret = prof_dump_write(propagate_err, buf);
1056 prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) {
1057 malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
1059 malloc_mutex_lock(tsdn, tctx->gctx->lock);
1061 switch (tctx->state) {
1062 case prof_tctx_state_initializing:
1063 malloc_mutex_unlock(tsdn, tctx->gctx->lock);
1065 case prof_tctx_state_nominal:
1066 tctx->state = prof_tctx_state_dumping;
1067 malloc_mutex_unlock(tsdn, tctx->gctx->lock);
1069 memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
1071 tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1072 tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1073 if (opt_prof_accum) {
1074 tdata->cnt_summed.accumobjs +=
1075 tctx->dump_cnts.accumobjs;
1076 tdata->cnt_summed.accumbytes +=
1077 tctx->dump_cnts.accumbytes;
1080 case prof_tctx_state_dumping:
1081 case prof_tctx_state_purgatory:
1087 prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) {
1088 malloc_mutex_assert_owner(tsdn, gctx->lock);
1090 gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1091 gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1092 if (opt_prof_accum) {
1093 gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
1094 gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
1098 static prof_tctx_t *
1099 prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
1100 tsdn_t *tsdn = (tsdn_t *)arg;
1102 malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
1104 switch (tctx->state) {
1105 case prof_tctx_state_nominal:
1106 /* New since dumping started; ignore. */
1108 case prof_tctx_state_dumping:
1109 case prof_tctx_state_purgatory:
1110 prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx);
1119 struct prof_tctx_dump_iter_arg_s {
1124 static prof_tctx_t *
1125 prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) {
1126 struct prof_tctx_dump_iter_arg_s *arg =
1127 (struct prof_tctx_dump_iter_arg_s *)opaque;
1129 malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock);
1131 switch (tctx->state) {
1132 case prof_tctx_state_initializing:
1133 case prof_tctx_state_nominal:
1134 /* Not captured by this dump. */
1136 case prof_tctx_state_dumping:
1137 case prof_tctx_state_purgatory:
1138 if (prof_dump_printf(arg->propagate_err,
1139 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": "
1140 "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs,
1141 tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs,
1142 tctx->dump_cnts.accumbytes)) {
1152 static prof_tctx_t *
1153 prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
1154 tsdn_t *tsdn = (tsdn_t *)arg;
1157 malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
1159 switch (tctx->state) {
1160 case prof_tctx_state_nominal:
1161 /* New since dumping started; ignore. */
1163 case prof_tctx_state_dumping:
1164 tctx->state = prof_tctx_state_nominal;
1166 case prof_tctx_state_purgatory:
1179 prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) {
1180 cassert(config_prof);
1182 malloc_mutex_lock(tsdn, gctx->lock);
1185 * Increment nlimbo so that gctx won't go away before dump.
1186 * Additionally, link gctx into the dump list so that it is included in
1187 * prof_dump()'s second pass.
1190 gctx_tree_insert(gctxs, gctx);
1192 memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
1194 malloc_mutex_unlock(tsdn, gctx->lock);
1197 struct prof_gctx_merge_iter_arg_s {
1202 static prof_gctx_t *
1203 prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
1204 struct prof_gctx_merge_iter_arg_s *arg =
1205 (struct prof_gctx_merge_iter_arg_s *)opaque;
1207 malloc_mutex_lock(arg->tsdn, gctx->lock);
1208 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter,
1210 if (gctx->cnt_summed.curobjs != 0) {
1213 malloc_mutex_unlock(arg->tsdn, gctx->lock);
1219 prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) {
1220 prof_tdata_t *tdata = prof_tdata_get(tsd, false);
1224 * Standard tree iteration won't work here, because as soon as we
1225 * decrement gctx->nlimbo and unlock gctx, another thread can
1226 * concurrently destroy it, which will corrupt the tree. Therefore,
1227 * tear down the tree one node at a time during iteration.
1229 while ((gctx = gctx_tree_first(gctxs)) != NULL) {
1230 gctx_tree_remove(gctxs, gctx);
1231 malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
1237 prof_tctx_t *to_destroy =
1238 tctx_tree_iter(&gctx->tctxs, next,
1239 prof_tctx_finish_iter,
1240 (void *)tsd_tsdn(tsd));
1241 if (to_destroy != NULL) {
1242 next = tctx_tree_next(&gctx->tctxs,
1244 tctx_tree_remove(&gctx->tctxs,
1246 idalloctm(tsd_tsdn(tsd), to_destroy,
1247 NULL, NULL, true, true);
1251 } while (next != NULL);
1254 if (prof_gctx_should_destroy(gctx)) {
1256 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
1257 prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
1259 malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
1264 struct prof_tdata_merge_iter_arg_s {
1269 static prof_tdata_t *
1270 prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
1272 struct prof_tdata_merge_iter_arg_s *arg =
1273 (struct prof_tdata_merge_iter_arg_s *)opaque;
1275 malloc_mutex_lock(arg->tsdn, tdata->lock);
1276 if (!tdata->expired) {
1283 tdata->dumping = true;
1284 memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
1285 for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
1287 prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);
1290 arg->cnt_all.curobjs += tdata->cnt_summed.curobjs;
1291 arg->cnt_all.curbytes += tdata->cnt_summed.curbytes;
1292 if (opt_prof_accum) {
1293 arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs;
1294 arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes;
1297 tdata->dumping = false;
1299 malloc_mutex_unlock(arg->tsdn, tdata->lock);
1304 static prof_tdata_t *
1305 prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
1307 bool propagate_err = *(bool *)arg;
1309 if (!tdata->dumping) {
1313 if (prof_dump_printf(propagate_err,
1314 " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n",
1315 tdata->thr_uid, tdata->cnt_summed.curobjs,
1316 tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,
1317 tdata->cnt_summed.accumbytes,
1318 (tdata->thread_name != NULL) ? " " : "",
1319 (tdata->thread_name != NULL) ? tdata->thread_name : "")) {
1326 prof_dump_header_impl(tsdn_t *tsdn, bool propagate_err,
1327 const prof_cnt_t *cnt_all) {
1330 if (prof_dump_printf(propagate_err,
1331 "heap_v2/%"FMTu64"\n"
1332 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1333 ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs,
1334 cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) {
1338 malloc_mutex_lock(tsdn, &tdatas_mtx);
1339 ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter,
1340 (void *)&propagate_err) != NULL);
1341 malloc_mutex_unlock(tsdn, &tdatas_mtx);
1344 prof_dump_header_t *JET_MUTABLE prof_dump_header = prof_dump_header_impl;
1347 prof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx,
1348 const prof_bt_t *bt, prof_gctx_tree_t *gctxs) {
1351 struct prof_tctx_dump_iter_arg_s prof_tctx_dump_iter_arg;
1353 cassert(config_prof);
1354 malloc_mutex_assert_owner(tsdn, gctx->lock);
1356 /* Avoid dumping such gctx's that have no useful data. */
1357 if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
1358 (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
1359 assert(gctx->cnt_summed.curobjs == 0);
1360 assert(gctx->cnt_summed.curbytes == 0);
1361 assert(gctx->cnt_summed.accumobjs == 0);
1362 assert(gctx->cnt_summed.accumbytes == 0);
1367 if (prof_dump_printf(propagate_err, "@")) {
1371 for (i = 0; i < bt->len; i++) {
1372 if (prof_dump_printf(propagate_err, " %#"FMTxPTR,
1373 (uintptr_t)bt->vec[i])) {
1379 if (prof_dump_printf(propagate_err,
1381 " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
1382 gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes,
1383 gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) {
1388 prof_tctx_dump_iter_arg.tsdn = tsdn;
1389 prof_tctx_dump_iter_arg.propagate_err = propagate_err;
1390 if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter,
1391 (void *)&prof_tctx_dump_iter_arg) != NULL) {
1402 JEMALLOC_FORMAT_PRINTF(1, 2)
1404 prof_open_maps(const char *format, ...) {
1407 char filename[PATH_MAX + 1];
1409 va_start(ap, format);
1410 malloc_vsnprintf(filename, sizeof(filename), format, ap);
1413 #if defined(O_CLOEXEC)
1414 mfd = open(filename, O_RDONLY | O_CLOEXEC);
1416 mfd = open(filename, O_RDONLY);
1418 fcntl(mfd, F_SETFD, fcntl(mfd, F_GETFD) | FD_CLOEXEC);
1429 return GetCurrentProcessId();
1436 prof_dump_maps(bool propagate_err) {
1440 cassert(config_prof);
1442 mfd = prof_open_maps("/proc/curproc/map");
1443 #elif defined(_WIN32)
1444 mfd = -1; // Not implemented
1447 int pid = prof_getpid();
1449 mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid);
1451 mfd = prof_open_maps("/proc/%d/maps", pid);
1458 if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") &&
1465 prof_dump_buf_end += nread;
1466 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
1467 /* Make space in prof_dump_buf before read(). */
1468 if (prof_dump_flush(propagate_err) &&
1474 nread = malloc_read_fd(mfd,
1475 &prof_dump_buf[prof_dump_buf_end], PROF_DUMP_BUFSIZE
1476 - prof_dump_buf_end);
1477 } while (nread > 0);
1492 * See prof_sample_threshold_update() comment for why the body of this function
1493 * is conditionally compiled.
1496 prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx,
1497 const char *filename) {
1498 #ifdef JEMALLOC_PROF
1500 * Scaling is equivalent AdjustSamples() in jeprof, but the result may
1501 * differ slightly from what jeprof reports, because here we scale the
1502 * summary values, whereas jeprof scales each context individually and
1503 * reports the sums of the scaled values.
1505 if (cnt_all->curbytes != 0) {
1506 double sample_period = (double)((uint64_t)1 << lg_prof_sample);
1507 double ratio = (((double)cnt_all->curbytes) /
1508 (double)cnt_all->curobjs) / sample_period;
1509 double scale_factor = 1.0 / (1.0 - exp(-ratio));
1510 uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes)
1512 uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) *
1515 malloc_printf("<jemalloc>: Leak approximation summary: ~%"FMTu64
1516 " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n",
1517 curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs !=
1518 1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : "");
1520 "<jemalloc>: Run jeprof on \"%s\" for leak detail\n",
1526 struct prof_gctx_dump_iter_arg_s {
1531 static prof_gctx_t *
1532 prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
1534 struct prof_gctx_dump_iter_arg_s *arg =
1535 (struct prof_gctx_dump_iter_arg_s *)opaque;
1537 malloc_mutex_lock(arg->tsdn, gctx->lock);
1539 if (prof_dump_gctx(arg->tsdn, arg->propagate_err, gctx, &gctx->bt,
1547 malloc_mutex_unlock(arg->tsdn, gctx->lock);
1552 prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata,
1553 struct prof_tdata_merge_iter_arg_s *prof_tdata_merge_iter_arg,
1554 struct prof_gctx_merge_iter_arg_s *prof_gctx_merge_iter_arg,
1555 prof_gctx_tree_t *gctxs) {
1562 prof_enter(tsd, tdata);
1565 * Put gctx's in limbo and clear their counters in preparation for
1568 gctx_tree_new(gctxs);
1569 for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) {
1570 prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, gctxs);
1574 * Iterate over tdatas, and for the non-expired ones snapshot their tctx
1575 * stats and merge them into the associated gctx's.
1577 prof_tdata_merge_iter_arg->tsdn = tsd_tsdn(tsd);
1578 memset(&prof_tdata_merge_iter_arg->cnt_all, 0, sizeof(prof_cnt_t));
1579 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1580 tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter,
1581 (void *)prof_tdata_merge_iter_arg);
1582 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1584 /* Merge tctx stats into gctx's. */
1585 prof_gctx_merge_iter_arg->tsdn = tsd_tsdn(tsd);
1586 prof_gctx_merge_iter_arg->leak_ngctx = 0;
1587 gctx_tree_iter(gctxs, NULL, prof_gctx_merge_iter,
1588 (void *)prof_gctx_merge_iter_arg);
1590 prof_leave(tsd, tdata);
1594 prof_dump_file(tsd_t *tsd, bool propagate_err, const char *filename,
1595 bool leakcheck, prof_tdata_t *tdata,
1596 struct prof_tdata_merge_iter_arg_s *prof_tdata_merge_iter_arg,
1597 struct prof_gctx_merge_iter_arg_s *prof_gctx_merge_iter_arg,
1598 struct prof_gctx_dump_iter_arg_s *prof_gctx_dump_iter_arg,
1599 prof_gctx_tree_t *gctxs) {
1600 /* Create dump file. */
1601 if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) {
1605 /* Dump profile header. */
1606 if (prof_dump_header(tsd_tsdn(tsd), propagate_err,
1607 &prof_tdata_merge_iter_arg->cnt_all)) {
1608 goto label_write_error;
1611 /* Dump per gctx profile stats. */
1612 prof_gctx_dump_iter_arg->tsdn = tsd_tsdn(tsd);
1613 prof_gctx_dump_iter_arg->propagate_err = propagate_err;
1614 if (gctx_tree_iter(gctxs, NULL, prof_gctx_dump_iter,
1615 (void *)prof_gctx_dump_iter_arg) != NULL) {
1616 goto label_write_error;
1619 /* Dump /proc/<pid>/maps if possible. */
1620 if (prof_dump_maps(propagate_err)) {
1621 goto label_write_error;
1624 if (prof_dump_close(propagate_err)) {
1630 prof_dump_close(propagate_err);
1635 prof_dump(tsd_t *tsd, bool propagate_err, const char *filename,
1637 cassert(config_prof);
1638 assert(tsd_reentrancy_level_get(tsd) == 0);
1640 prof_tdata_t * tdata = prof_tdata_get(tsd, true);
1641 if (tdata == NULL) {
1645 pre_reentrancy(tsd, NULL);
1646 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
1648 prof_gctx_tree_t gctxs;
1649 struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg;
1650 struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg;
1651 struct prof_gctx_dump_iter_arg_s prof_gctx_dump_iter_arg;
1652 prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,
1653 &prof_gctx_merge_iter_arg, &gctxs);
1654 bool err = prof_dump_file(tsd, propagate_err, filename, leakcheck, tdata,
1655 &prof_tdata_merge_iter_arg, &prof_gctx_merge_iter_arg,
1656 &prof_gctx_dump_iter_arg, &gctxs);
1657 prof_gctx_finish(tsd, &gctxs);
1659 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
1660 post_reentrancy(tsd);
1667 prof_leakcheck(&prof_tdata_merge_iter_arg.cnt_all,
1668 prof_gctx_merge_iter_arg.leak_ngctx, filename);
1675 prof_cnt_all(uint64_t *curobjs, uint64_t *curbytes, uint64_t *accumobjs,
1676 uint64_t *accumbytes) {
1678 prof_tdata_t *tdata;
1679 struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg;
1680 struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg;
1681 prof_gctx_tree_t gctxs;
1684 tdata = prof_tdata_get(tsd, false);
1685 if (tdata == NULL) {
1686 if (curobjs != NULL) {
1689 if (curbytes != NULL) {
1692 if (accumobjs != NULL) {
1695 if (accumbytes != NULL) {
1701 prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,
1702 &prof_gctx_merge_iter_arg, &gctxs);
1703 prof_gctx_finish(tsd, &gctxs);
1705 if (curobjs != NULL) {
1706 *curobjs = prof_tdata_merge_iter_arg.cnt_all.curobjs;
1708 if (curbytes != NULL) {
1709 *curbytes = prof_tdata_merge_iter_arg.cnt_all.curbytes;
1711 if (accumobjs != NULL) {
1712 *accumobjs = prof_tdata_merge_iter_arg.cnt_all.accumobjs;
1714 if (accumbytes != NULL) {
1715 *accumbytes = prof_tdata_merge_iter_arg.cnt_all.accumbytes;
1720 #define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1)
1721 #define VSEQ_INVALID UINT64_C(0xffffffffffffffff)
1723 prof_dump_filename(char *filename, char v, uint64_t vseq) {
1724 cassert(config_prof);
1726 if (vseq != VSEQ_INVALID) {
1727 /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */
1728 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1729 "%s.%d.%"FMTu64".%c%"FMTu64".heap",
1730 opt_prof_prefix, prof_getpid(), prof_dump_seq, v, vseq);
1732 /* "<prefix>.<pid>.<seq>.<v>.heap" */
1733 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1734 "%s.%d.%"FMTu64".%c.heap",
1735 opt_prof_prefix, prof_getpid(), prof_dump_seq, v);
1743 char filename[DUMP_FILENAME_BUFSIZE];
1745 cassert(config_prof);
1746 assert(opt_prof_final);
1747 assert(opt_prof_prefix[0] != '\0');
1753 assert(tsd_reentrancy_level_get(tsd) == 0);
1755 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1756 prof_dump_filename(filename, 'f', VSEQ_INVALID);
1757 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1758 prof_dump(tsd, false, filename, opt_prof_leak);
1762 prof_accum_init(tsdn_t *tsdn, prof_accum_t *prof_accum) {
1763 cassert(config_prof);
1765 #ifndef JEMALLOC_ATOMIC_U64
1766 if (malloc_mutex_init(&prof_accum->mtx, "prof_accum",
1767 WITNESS_RANK_PROF_ACCUM, malloc_mutex_rank_exclusive)) {
1770 prof_accum->accumbytes = 0;
1772 atomic_store_u64(&prof_accum->accumbytes, 0, ATOMIC_RELAXED);
1778 prof_idump(tsdn_t *tsdn) {
1780 prof_tdata_t *tdata;
1782 cassert(config_prof);
1784 if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) {
1787 tsd = tsdn_tsd(tsdn);
1788 if (tsd_reentrancy_level_get(tsd) > 0) {
1792 tdata = prof_tdata_get(tsd, false);
1793 if (tdata == NULL) {
1797 tdata->enq_idump = true;
1801 if (opt_prof_prefix[0] != '\0') {
1802 char filename[PATH_MAX + 1];
1803 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1804 prof_dump_filename(filename, 'i', prof_dump_iseq);
1806 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1807 prof_dump(tsd, false, filename, false);
1812 prof_mdump(tsd_t *tsd, const char *filename) {
1813 cassert(config_prof);
1814 assert(tsd_reentrancy_level_get(tsd) == 0);
1816 if (!opt_prof || !prof_booted) {
1819 char filename_buf[DUMP_FILENAME_BUFSIZE];
1820 if (filename == NULL) {
1821 /* No filename specified, so automatically generate one. */
1822 if (opt_prof_prefix[0] == '\0') {
1825 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1826 prof_dump_filename(filename_buf, 'm', prof_dump_mseq);
1828 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
1829 filename = filename_buf;
1831 return prof_dump(tsd, true, filename, false);
1835 prof_gdump(tsdn_t *tsdn) {
1837 prof_tdata_t *tdata;
1839 cassert(config_prof);
1841 if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) {
1844 tsd = tsdn_tsd(tsdn);
1845 if (tsd_reentrancy_level_get(tsd) > 0) {
1849 tdata = prof_tdata_get(tsd, false);
1850 if (tdata == NULL) {
1854 tdata->enq_gdump = true;
1858 if (opt_prof_prefix[0] != '\0') {
1859 char filename[DUMP_FILENAME_BUFSIZE];
1860 malloc_mutex_lock(tsdn, &prof_dump_seq_mtx);
1861 prof_dump_filename(filename, 'u', prof_dump_useq);
1863 malloc_mutex_unlock(tsdn, &prof_dump_seq_mtx);
1864 prof_dump(tsd, false, filename, false);
1869 prof_bt_hash(const void *key, size_t r_hash[2]) {
1870 prof_bt_t *bt = (prof_bt_t *)key;
1872 cassert(config_prof);
1874 hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
1878 prof_bt_keycomp(const void *k1, const void *k2) {
1879 const prof_bt_t *bt1 = (prof_bt_t *)k1;
1880 const prof_bt_t *bt2 = (prof_bt_t *)k2;
1882 cassert(config_prof);
1884 if (bt1->len != bt2->len) {
1887 return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
1891 prof_thr_uid_alloc(tsdn_t *tsdn) {
1894 malloc_mutex_lock(tsdn, &next_thr_uid_mtx);
1895 thr_uid = next_thr_uid;
1897 malloc_mutex_unlock(tsdn, &next_thr_uid_mtx);
1902 static prof_tdata_t *
1903 prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
1904 char *thread_name, bool active) {
1905 prof_tdata_t *tdata;
1907 cassert(config_prof);
1909 /* Initialize an empty cache for this thread. */
1910 tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),
1911 sz_size2index(sizeof(prof_tdata_t)), false, NULL, true,
1912 arena_get(TSDN_NULL, 0, true), true);
1913 if (tdata == NULL) {
1917 tdata->lock = prof_tdata_mutex_choose(thr_uid);
1918 tdata->thr_uid = thr_uid;
1919 tdata->thr_discrim = thr_discrim;
1920 tdata->thread_name = thread_name;
1921 tdata->attached = true;
1922 tdata->expired = false;
1923 tdata->tctx_uid_next = 0;
1925 if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,
1927 idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
1931 tdata->prng_state = (uint64_t)(uintptr_t)tdata;
1932 prof_sample_threshold_update(tdata);
1935 tdata->enq_idump = false;
1936 tdata->enq_gdump = false;
1938 tdata->dumping = false;
1939 tdata->active = active;
1941 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1942 tdata_tree_insert(&tdatas, tdata);
1943 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1949 prof_tdata_init(tsd_t *tsd) {
1950 return prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd_tsdn(tsd)), 0,
1951 NULL, prof_thread_active_init_get(tsd_tsdn(tsd)));
1955 prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) {
1956 if (tdata->attached && !even_if_attached) {
1959 if (ckh_count(&tdata->bt2tctx) != 0) {
1966 prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
1967 bool even_if_attached) {
1968 malloc_mutex_assert_owner(tsdn, tdata->lock);
1970 return prof_tdata_should_destroy_unlocked(tdata, even_if_attached);
1974 prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
1975 bool even_if_attached) {
1976 malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx);
1978 tdata_tree_remove(&tdatas, tdata);
1980 assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
1982 if (tdata->thread_name != NULL) {
1983 idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
1986 ckh_delete(tsd, &tdata->bt2tctx);
1987 idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
1991 prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) {
1992 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
1993 prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
1994 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
1998 prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) {
2001 malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
2002 if (tdata->attached) {
2003 destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,
2006 * Only detach if !destroy_tdata, because detaching would allow
2007 * another thread to win the race to destroy tdata.
2009 if (!destroy_tdata) {
2010 tdata->attached = false;
2012 tsd_prof_tdata_set(tsd, NULL);
2014 destroy_tdata = false;
2016 malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
2017 if (destroy_tdata) {
2018 prof_tdata_destroy(tsd, tdata, true);
2023 prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) {
2024 uint64_t thr_uid = tdata->thr_uid;
2025 uint64_t thr_discrim = tdata->thr_discrim + 1;
2026 char *thread_name = (tdata->thread_name != NULL) ?
2027 prof_thread_name_alloc(tsd_tsdn(tsd), tdata->thread_name) : NULL;
2028 bool active = tdata->active;
2030 prof_tdata_detach(tsd, tdata);
2031 return prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name,
2036 prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) {
2039 malloc_mutex_lock(tsdn, tdata->lock);
2040 if (!tdata->expired) {
2041 tdata->expired = true;
2042 destroy_tdata = tdata->attached ? false :
2043 prof_tdata_should_destroy(tsdn, tdata, false);
2045 destroy_tdata = false;
2047 malloc_mutex_unlock(tsdn, tdata->lock);
2049 return destroy_tdata;
2052 static prof_tdata_t *
2053 prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
2055 tsdn_t *tsdn = (tsdn_t *)arg;
2057 return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL);
2061 prof_reset(tsd_t *tsd, size_t lg_sample) {
2064 assert(lg_sample < (sizeof(uint64_t) << 3));
2066 malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
2067 malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
2069 lg_prof_sample = lg_sample;
2073 prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
2074 prof_tdata_reset_iter, (void *)tsd);
2075 if (to_destroy != NULL) {
2076 next = tdata_tree_next(&tdatas, to_destroy);
2077 prof_tdata_destroy_locked(tsd, to_destroy, false);
2081 } while (next != NULL);
2083 malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
2084 malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
2088 prof_tdata_cleanup(tsd_t *tsd) {
2089 prof_tdata_t *tdata;
2095 tdata = tsd_prof_tdata_get(tsd);
2096 if (tdata != NULL) {
2097 prof_tdata_detach(tsd, tdata);
2102 prof_active_get(tsdn_t *tsdn) {
2103 bool prof_active_current;
2105 malloc_mutex_lock(tsdn, &prof_active_mtx);
2106 prof_active_current = prof_active;
2107 malloc_mutex_unlock(tsdn, &prof_active_mtx);
2108 return prof_active_current;
2112 prof_active_set(tsdn_t *tsdn, bool active) {
2113 bool prof_active_old;
2115 malloc_mutex_lock(tsdn, &prof_active_mtx);
2116 prof_active_old = prof_active;
2117 prof_active = active;
2118 malloc_mutex_unlock(tsdn, &prof_active_mtx);
2119 return prof_active_old;
2123 prof_thread_name_get(tsd_t *tsd) {
2124 prof_tdata_t *tdata;
2126 tdata = prof_tdata_get(tsd, true);
2127 if (tdata == NULL) {
2130 return (tdata->thread_name != NULL ? tdata->thread_name : "");
2134 prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name) {
2138 if (thread_name == NULL) {
2142 size = strlen(thread_name) + 1;
2147 ret = iallocztm(tsdn, size, sz_size2index(size), false, NULL, true,
2148 arena_get(TSDN_NULL, 0, true), true);
2152 memcpy(ret, thread_name, size);
2157 prof_thread_name_set(tsd_t *tsd, const char *thread_name) {
2158 prof_tdata_t *tdata;
2162 tdata = prof_tdata_get(tsd, true);
2163 if (tdata == NULL) {
2167 /* Validate input. */
2168 if (thread_name == NULL) {
2171 for (i = 0; thread_name[i] != '\0'; i++) {
2172 char c = thread_name[i];
2173 if (!isgraph(c) && !isblank(c)) {
2178 s = prof_thread_name_alloc(tsd_tsdn(tsd), thread_name);
2183 if (tdata->thread_name != NULL) {
2184 idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
2186 tdata->thread_name = NULL;
2188 if (strlen(s) > 0) {
2189 tdata->thread_name = s;
2195 prof_thread_active_get(tsd_t *tsd) {
2196 prof_tdata_t *tdata;
2198 tdata = prof_tdata_get(tsd, true);
2199 if (tdata == NULL) {
2202 return tdata->active;
2206 prof_thread_active_set(tsd_t *tsd, bool active) {
2207 prof_tdata_t *tdata;
2209 tdata = prof_tdata_get(tsd, true);
2210 if (tdata == NULL) {
2213 tdata->active = active;
2218 prof_thread_active_init_get(tsdn_t *tsdn) {
2221 malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx);
2222 active_init = prof_thread_active_init;
2223 malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx);
2228 prof_thread_active_init_set(tsdn_t *tsdn, bool active_init) {
2229 bool active_init_old;
2231 malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx);
2232 active_init_old = prof_thread_active_init;
2233 prof_thread_active_init = active_init;
2234 malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx);
2235 return active_init_old;
2239 prof_gdump_get(tsdn_t *tsdn) {
2240 bool prof_gdump_current;
2242 malloc_mutex_lock(tsdn, &prof_gdump_mtx);
2243 prof_gdump_current = prof_gdump_val;
2244 malloc_mutex_unlock(tsdn, &prof_gdump_mtx);
2245 return prof_gdump_current;
2249 prof_gdump_set(tsdn_t *tsdn, bool gdump) {
2250 bool prof_gdump_old;
2252 malloc_mutex_lock(tsdn, &prof_gdump_mtx);
2253 prof_gdump_old = prof_gdump_val;
2254 prof_gdump_val = gdump;
2255 malloc_mutex_unlock(tsdn, &prof_gdump_mtx);
2256 return prof_gdump_old;
2261 cassert(config_prof);
2263 memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT,
2264 sizeof(PROF_PREFIX_DEFAULT));
2269 cassert(config_prof);
2272 * opt_prof must be in its final state before any arenas are
2273 * initialized, so this function must be executed early.
2276 if (opt_prof_leak && !opt_prof) {
2278 * Enable opt_prof, but in such a way that profiles are never
2279 * automatically dumped.
2282 opt_prof_gdump = false;
2283 } else if (opt_prof) {
2284 if (opt_lg_prof_interval >= 0) {
2285 prof_interval = (((uint64_t)1U) <<
2286 opt_lg_prof_interval);
2292 prof_boot2(tsd_t *tsd) {
2293 cassert(config_prof);
2298 lg_prof_sample = opt_lg_prof_sample;
2300 prof_active = opt_prof_active;
2301 if (malloc_mutex_init(&prof_active_mtx, "prof_active",
2302 WITNESS_RANK_PROF_ACTIVE, malloc_mutex_rank_exclusive)) {
2306 prof_gdump_val = opt_prof_gdump;
2307 if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump",
2308 WITNESS_RANK_PROF_GDUMP, malloc_mutex_rank_exclusive)) {
2312 prof_thread_active_init = opt_prof_thread_active_init;
2313 if (malloc_mutex_init(&prof_thread_active_init_mtx,
2314 "prof_thread_active_init",
2315 WITNESS_RANK_PROF_THREAD_ACTIVE_INIT,
2316 malloc_mutex_rank_exclusive)) {
2320 if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
2324 if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx",
2325 WITNESS_RANK_PROF_BT2GCTX, malloc_mutex_rank_exclusive)) {
2329 tdata_tree_new(&tdatas);
2330 if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas",
2331 WITNESS_RANK_PROF_TDATAS, malloc_mutex_rank_exclusive)) {
2336 if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid",
2337 WITNESS_RANK_PROF_NEXT_THR_UID, malloc_mutex_rank_exclusive)) {
2341 if (malloc_mutex_init(&prof_dump_seq_mtx, "prof_dump_seq",
2342 WITNESS_RANK_PROF_DUMP_SEQ, malloc_mutex_rank_exclusive)) {
2345 if (malloc_mutex_init(&prof_dump_mtx, "prof_dump",
2346 WITNESS_RANK_PROF_DUMP, malloc_mutex_rank_exclusive)) {
2350 if (opt_prof_final && opt_prof_prefix[0] != '\0' &&
2351 atexit(prof_fdump) != 0) {
2352 malloc_write("<jemalloc>: Error in atexit()\n");
2358 gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),
2359 b0get(), PROF_NCTX_LOCKS * sizeof(malloc_mutex_t),
2361 if (gctx_locks == NULL) {
2364 for (i = 0; i < PROF_NCTX_LOCKS; i++) {
2365 if (malloc_mutex_init(&gctx_locks[i], "prof_gctx",
2366 WITNESS_RANK_PROF_GCTX,
2367 malloc_mutex_rank_exclusive)) {
2372 tdata_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),
2373 b0get(), PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t),
2375 if (tdata_locks == NULL) {
2378 for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
2379 if (malloc_mutex_init(&tdata_locks[i], "prof_tdata",
2380 WITNESS_RANK_PROF_TDATA,
2381 malloc_mutex_rank_exclusive)) {
2387 #ifdef JEMALLOC_PROF_LIBGCC
2389 * Cause the backtracing machinery to allocate its internal state
2390 * before enabling profiling.
2392 _Unwind_Backtrace(prof_unwind_init_callback, NULL);
2401 prof_prefork0(tsdn_t *tsdn) {
2402 if (config_prof && opt_prof) {
2405 malloc_mutex_prefork(tsdn, &prof_dump_mtx);
2406 malloc_mutex_prefork(tsdn, &bt2gctx_mtx);
2407 malloc_mutex_prefork(tsdn, &tdatas_mtx);
2408 for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
2409 malloc_mutex_prefork(tsdn, &tdata_locks[i]);
2411 for (i = 0; i < PROF_NCTX_LOCKS; i++) {
2412 malloc_mutex_prefork(tsdn, &gctx_locks[i]);
2418 prof_prefork1(tsdn_t *tsdn) {
2419 if (config_prof && opt_prof) {
2420 malloc_mutex_prefork(tsdn, &prof_active_mtx);
2421 malloc_mutex_prefork(tsdn, &prof_dump_seq_mtx);
2422 malloc_mutex_prefork(tsdn, &prof_gdump_mtx);
2423 malloc_mutex_prefork(tsdn, &next_thr_uid_mtx);
2424 malloc_mutex_prefork(tsdn, &prof_thread_active_init_mtx);
2429 prof_postfork_parent(tsdn_t *tsdn) {
2430 if (config_prof && opt_prof) {
2433 malloc_mutex_postfork_parent(tsdn,
2434 &prof_thread_active_init_mtx);
2435 malloc_mutex_postfork_parent(tsdn, &next_thr_uid_mtx);
2436 malloc_mutex_postfork_parent(tsdn, &prof_gdump_mtx);
2437 malloc_mutex_postfork_parent(tsdn, &prof_dump_seq_mtx);
2438 malloc_mutex_postfork_parent(tsdn, &prof_active_mtx);
2439 for (i = 0; i < PROF_NCTX_LOCKS; i++) {
2440 malloc_mutex_postfork_parent(tsdn, &gctx_locks[i]);
2442 for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
2443 malloc_mutex_postfork_parent(tsdn, &tdata_locks[i]);
2445 malloc_mutex_postfork_parent(tsdn, &tdatas_mtx);
2446 malloc_mutex_postfork_parent(tsdn, &bt2gctx_mtx);
2447 malloc_mutex_postfork_parent(tsdn, &prof_dump_mtx);
2452 prof_postfork_child(tsdn_t *tsdn) {
2453 if (config_prof && opt_prof) {
2456 malloc_mutex_postfork_child(tsdn, &prof_thread_active_init_mtx);
2457 malloc_mutex_postfork_child(tsdn, &next_thr_uid_mtx);
2458 malloc_mutex_postfork_child(tsdn, &prof_gdump_mtx);
2459 malloc_mutex_postfork_child(tsdn, &prof_dump_seq_mtx);
2460 malloc_mutex_postfork_child(tsdn, &prof_active_mtx);
2461 for (i = 0; i < PROF_NCTX_LOCKS; i++) {
2462 malloc_mutex_postfork_child(tsdn, &gctx_locks[i]);
2464 for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
2465 malloc_mutex_postfork_child(tsdn, &tdata_locks[i]);
2467 malloc_mutex_postfork_child(tsdn, &tdatas_mtx);
2468 malloc_mutex_postfork_child(tsdn, &bt2gctx_mtx);
2469 malloc_mutex_postfork_child(tsdn, &prof_dump_mtx);
2473 /******************************************************************************/