1 #ifndef JEMALLOC_INTERNAL_ARENA_STATS_H
2 #define JEMALLOC_INTERNAL_ARENA_STATS_H
4 #include "jemalloc/internal/atomic.h"
5 #include "jemalloc/internal/mutex.h"
6 #include "jemalloc/internal/mutex_prof.h"
7 #include "jemalloc/internal/sc.h"
9 JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
12 * In those architectures that support 64-bit atomics, we use atomic updates for
13 * our 64-bit values. Otherwise, we use a plain uint64_t and synchronize
16 #ifdef JEMALLOC_ATOMIC_U64
17 typedef atomic_u64_t arena_stats_u64_t;
19 /* Must hold the arena stats mutex while reading atomically. */
20 typedef uint64_t arena_stats_u64_t;
23 typedef struct arena_stats_large_s arena_stats_large_t;
24 struct arena_stats_large_s {
26 * Total number of allocation/deallocation requests served directly by
29 arena_stats_u64_t nmalloc;
30 arena_stats_u64_t ndalloc;
33 * Number of allocation requests that correspond to this size class.
34 * This includes requests served by tcache, though tcache only
35 * periodically merges into this counter.
37 arena_stats_u64_t nrequests; /* Partially derived. */
39 * Number of tcache fills / flushes for large (similarly, periodically
40 * merged). Note that there is no large tcache batch-fill currently
41 * (i.e. only fill 1 at a time); however flush may be batched.
43 arena_stats_u64_t nfills; /* Partially derived. */
44 arena_stats_u64_t nflushes; /* Partially derived. */
46 /* Current number of allocations of this size class. */
47 size_t curlextents; /* Derived. */
50 typedef struct arena_stats_decay_s arena_stats_decay_t;
51 struct arena_stats_decay_s {
52 /* Total number of purge sweeps. */
53 arena_stats_u64_t npurge;
54 /* Total number of madvise calls made. */
55 arena_stats_u64_t nmadvise;
56 /* Total number of pages purged. */
57 arena_stats_u64_t purged;
60 typedef struct arena_stats_extents_s arena_stats_extents_t;
61 struct arena_stats_extents_s {
63 * Stats for a given index in the range [0, SC_NPSIZES] in an extents_t.
64 * We track both bytes and # of extents: two extents in the same bucket
65 * may have different sizes if adjacent size classes differ by more than
66 * a page, so bytes cannot always be derived from # of extents.
69 atomic_zu_t dirty_bytes;
71 atomic_zu_t muzzy_bytes;
72 atomic_zu_t nretained;
73 atomic_zu_t retained_bytes;
77 * Arena stats. Note that fields marked "derived" are not directly maintained
78 * within the arena code; rather their values are derived during stats merge
81 typedef struct arena_stats_s arena_stats_t;
82 struct arena_stats_s {
83 #ifndef JEMALLOC_ATOMIC_U64
87 /* Number of bytes currently mapped, excluding retained memory. */
88 atomic_zu_t mapped; /* Partially derived. */
91 * Number of unused virtual memory bytes currently retained. Retained
92 * bytes are technically mapped (though always decommitted or purged),
93 * but they are excluded from the mapped statistic (above).
95 atomic_zu_t retained; /* Derived. */
97 /* Number of extent_t structs allocated by base, but not being used. */
98 atomic_zu_t extent_avail;
100 arena_stats_decay_t decay_dirty;
101 arena_stats_decay_t decay_muzzy;
103 atomic_zu_t base; /* Derived. */
104 atomic_zu_t internal;
105 atomic_zu_t resident; /* Derived. */
106 atomic_zu_t metadata_thp;
108 atomic_zu_t allocated_large; /* Derived. */
109 arena_stats_u64_t nmalloc_large; /* Derived. */
110 arena_stats_u64_t ndalloc_large; /* Derived. */
111 arena_stats_u64_t nfills_large; /* Derived. */
112 arena_stats_u64_t nflushes_large; /* Derived. */
113 arena_stats_u64_t nrequests_large; /* Derived. */
115 /* VM space had to be leaked (undocumented). Normally 0. */
116 atomic_zu_t abandoned_vm;
118 /* Number of bytes cached in tcache associated with this arena. */
119 atomic_zu_t tcache_bytes; /* Derived. */
121 mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes];
123 /* One element for each large size class. */
124 arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
131 arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) {
133 for (size_t i = 0; i < sizeof(arena_stats_t); i++) {
134 assert(((char *)arena_stats)[i] == 0);
137 #ifndef JEMALLOC_ATOMIC_U64
138 if (malloc_mutex_init(&arena_stats->mtx, "arena_stats",
139 WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) {
143 /* Memory is zeroed, so there is no need to clear stats. */
148 arena_stats_lock(tsdn_t *tsdn, arena_stats_t *arena_stats) {
149 #ifndef JEMALLOC_ATOMIC_U64
150 malloc_mutex_lock(tsdn, &arena_stats->mtx);
155 arena_stats_unlock(tsdn_t *tsdn, arena_stats_t *arena_stats) {
156 #ifndef JEMALLOC_ATOMIC_U64
157 malloc_mutex_unlock(tsdn, &arena_stats->mtx);
161 static inline uint64_t
162 arena_stats_read_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
163 arena_stats_u64_t *p) {
164 #ifdef JEMALLOC_ATOMIC_U64
165 return atomic_load_u64(p, ATOMIC_RELAXED);
167 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
173 arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
174 arena_stats_u64_t *p, uint64_t x) {
175 #ifdef JEMALLOC_ATOMIC_U64
176 atomic_fetch_add_u64(p, x, ATOMIC_RELAXED);
178 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
184 arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
185 arena_stats_u64_t *p, uint64_t x) {
186 #ifdef JEMALLOC_ATOMIC_U64
187 uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED);
190 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
192 assert(*p + x >= *p);
197 * Non-atomically sets *dst += src. *dst needs external synchronization.
198 * This lets us avoid the cost of a fetch_add when its unnecessary (note that
199 * the types here are atomic).
202 arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) {
203 #ifdef JEMALLOC_ATOMIC_U64
204 uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED);
205 atomic_store_u64(dst, src + cur_dst, ATOMIC_RELAXED);
212 arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
214 #ifdef JEMALLOC_ATOMIC_U64
215 return atomic_load_zu(p, ATOMIC_RELAXED);
217 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
218 return atomic_load_zu(p, ATOMIC_RELAXED);
223 arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
224 atomic_zu_t *p, size_t x) {
225 #ifdef JEMALLOC_ATOMIC_U64
226 atomic_fetch_add_zu(p, x, ATOMIC_RELAXED);
228 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
229 size_t cur = atomic_load_zu(p, ATOMIC_RELAXED);
230 atomic_store_zu(p, cur + x, ATOMIC_RELAXED);
235 arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
236 atomic_zu_t *p, size_t x) {
237 #ifdef JEMALLOC_ATOMIC_U64
238 size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED);
241 malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
242 size_t cur = atomic_load_zu(p, ATOMIC_RELAXED);
243 atomic_store_zu(p, cur - x, ATOMIC_RELAXED);
247 /* Like the _u64 variant, needs an externally synchronized *dst. */
249 arena_stats_accum_zu(atomic_zu_t *dst, size_t src) {
250 size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED);
251 atomic_store_zu(dst, src + cur_dst, ATOMIC_RELAXED);
255 arena_stats_large_flush_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
256 szind_t szind, uint64_t nrequests) {
257 arena_stats_lock(tsdn, arena_stats);
258 arena_stats_large_t *lstats = &arena_stats->lstats[szind - SC_NBINS];
259 arena_stats_add_u64(tsdn, arena_stats, &lstats->nrequests, nrequests);
260 arena_stats_add_u64(tsdn, arena_stats, &lstats->nflushes, 1);
261 arena_stats_unlock(tsdn, arena_stats);
265 arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) {
266 arena_stats_lock(tsdn, arena_stats);
267 arena_stats_add_zu(tsdn, arena_stats, &arena_stats->mapped, size);
268 arena_stats_unlock(tsdn, arena_stats);
271 #endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */