1 #ifndef JEMALLOC_INTERNAL_MUTEX_H
2 #define JEMALLOC_INTERNAL_MUTEX_H
4 #include "jemalloc/internal/atomic.h"
5 #include "jemalloc/internal/mutex_prof.h"
6 #include "jemalloc/internal/tsd.h"
7 #include "jemalloc/internal/witness.h"
10 /* Can only acquire one mutex of a given witness rank at a time. */
11 malloc_mutex_rank_exclusive,
13 * Can acquire multiple mutexes of the same witness rank, but in
14 * address-ascending order only.
16 malloc_mutex_address_ordered
17 } malloc_mutex_lock_order_t;
19 typedef struct malloc_mutex_s malloc_mutex_t;
20 struct malloc_mutex_s {
24 * prof_data is defined first to reduce cacheline
25 * bouncing: the data is not touched by the mutex holder
26 * during unlocking, while might be modified by
27 * contenders. Having it before the mutex itself could
28 * avoid prefetching a modified cacheline (for the
31 mutex_prof_data_t prof_data;
33 # if _WIN32_WINNT >= 0x0600
36 CRITICAL_SECTION lock;
38 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
40 #elif (defined(JEMALLOC_MUTEX_INIT_CB))
42 malloc_mutex_t *postponed_next;
47 * Hint flag to avoid exclusive cache line contention
53 * We only touch witness when configured w/ debug. However we
54 * keep the field in a union when !debug so that we don't have
55 * to pollute the code base with #ifdefs, while avoid paying the
58 #if !defined(JEMALLOC_DEBUG)
60 malloc_mutex_lock_order_t lock_order;
64 #if defined(JEMALLOC_DEBUG)
66 malloc_mutex_lock_order_t lock_order;
71 * Based on benchmark results, a fixed spin with this amount of retries works
72 * well for our critical sections.
74 #define MALLOC_MUTEX_MAX_SPIN 250
77 # if _WIN32_WINNT >= 0x0600
78 # define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock)
79 # define MALLOC_MUTEX_UNLOCK(m) ReleaseSRWLockExclusive(&(m)->lock)
80 # define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock))
82 # define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock)
83 # define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock)
84 # define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock))
86 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
87 # define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock)
88 # define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock)
89 # define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock))
91 # define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock)
92 # define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock)
93 # define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0)
96 #define LOCK_PROF_DATA_INITIALIZER \
97 {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \
98 ATOMIC_INIT(0), 0, NULL, 0}
101 # define MALLOC_MUTEX_INITIALIZER
102 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
103 # if defined(JEMALLOC_DEBUG)
104 # define MALLOC_MUTEX_INITIALIZER \
105 {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \
106 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
108 # define MALLOC_MUTEX_INITIALIZER \
109 {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \
110 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
112 #elif (defined(JEMALLOC_MUTEX_INIT_CB))
113 # if (defined(JEMALLOC_DEBUG))
114 # define MALLOC_MUTEX_INITIALIZER \
115 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \
116 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
118 # define MALLOC_MUTEX_INITIALIZER \
119 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \
120 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
124 # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
125 # if defined(JEMALLOC_DEBUG)
126 # define MALLOC_MUTEX_INITIALIZER \
127 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \
128 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
130 # define MALLOC_MUTEX_INITIALIZER \
131 {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \
132 WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
136 #ifdef JEMALLOC_LAZY_LOCK
137 extern bool isthreaded;
140 bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
141 witness_rank_t rank, malloc_mutex_lock_order_t lock_order);
142 void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
143 void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
144 void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
145 bool malloc_mutex_first_thread(void);
146 bool malloc_mutex_boot(void);
147 void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex);
149 void malloc_mutex_lock_slow(malloc_mutex_t *mutex);
152 malloc_mutex_lock_final(malloc_mutex_t *mutex) {
153 MALLOC_MUTEX_LOCK(mutex);
154 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
158 malloc_mutex_trylock_final(malloc_mutex_t *mutex) {
159 return MALLOC_MUTEX_TRYLOCK(mutex);
163 mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) {
165 mutex_prof_data_t *data = &mutex->prof_data;
167 if (data->prev_owner != tsdn) {
168 data->prev_owner = tsdn;
169 data->n_owner_switches++;
174 /* Trylock: return false if the lock is successfully acquired. */
176 malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
177 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
179 if (malloc_mutex_trylock_final(mutex)) {
180 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
183 mutex_owner_stats_update(tsdn, mutex);
185 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
190 /* Aggregate lock prof data. */
192 malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
193 nstime_add(&sum->tot_wait_time, &data->tot_wait_time);
194 if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) {
195 nstime_copy(&sum->max_wait_time, &data->max_wait_time);
198 sum->n_wait_times += data->n_wait_times;
199 sum->n_spin_acquired += data->n_spin_acquired;
201 if (sum->max_n_thds < data->max_n_thds) {
202 sum->max_n_thds = data->max_n_thds;
204 uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds,
206 uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32(
207 &data->n_waiting_thds, ATOMIC_RELAXED);
208 atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds,
210 sum->n_owner_switches += data->n_owner_switches;
211 sum->n_lock_ops += data->n_lock_ops;
215 malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
216 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
218 if (malloc_mutex_trylock_final(mutex)) {
219 malloc_mutex_lock_slow(mutex);
220 atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
222 mutex_owner_stats_update(tsdn, mutex);
224 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
228 malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
229 atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED);
230 witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
232 MALLOC_MUTEX_UNLOCK(mutex);
237 malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
238 witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
242 malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
243 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
246 /* Copy the prof data from mutex for processing. */
248 malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
249 malloc_mutex_t *mutex) {
250 mutex_prof_data_t *source = &mutex->prof_data;
251 /* Can only read holding the mutex. */
252 malloc_mutex_assert_owner(tsdn, mutex);
255 * Not *really* allowed (we shouldn't be doing non-atomic loads of
256 * atomic data), but the mutex protection makes this safe, and writing
257 * a member-for-member copy is tedious for this situation.
260 /* n_wait_thds is not reported (modified w/o locking). */
261 atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
265 malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data,
266 malloc_mutex_t *mutex) {
267 mutex_prof_data_t *source = &mutex->prof_data;
268 /* Can only read holding the mutex. */
269 malloc_mutex_assert_owner(tsdn, mutex);
271 nstime_add(&data->tot_wait_time, &source->tot_wait_time);
272 if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) {
273 nstime_copy(&data->max_wait_time, &source->max_wait_time);
275 data->n_wait_times += source->n_wait_times;
276 data->n_spin_acquired += source->n_spin_acquired;
277 if (data->max_n_thds < source->max_n_thds) {
278 data->max_n_thds = source->max_n_thds;
280 /* n_wait_thds is not reported. */
281 atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
282 data->n_owner_switches += source->n_owner_switches;
283 data->n_lock_ops += source->n_lock_ops;
286 #endif /* JEMALLOC_INTERNAL_MUTEX_H */