1 #ifndef JEMALLOC_INTERNAL_WITNESS_H
2 #define JEMALLOC_INTERNAL_WITNESS_H
4 #include "jemalloc/internal/ql.h"
6 /******************************************************************************/
8 /******************************************************************************/
11 * Witnesses with rank WITNESS_RANK_OMIT are completely ignored by the witness
15 #define WITNESS_RANK_OMIT 0U
17 #define WITNESS_RANK_MIN 1U
19 #define WITNESS_RANK_INIT 1U
20 #define WITNESS_RANK_CTL 1U
21 #define WITNESS_RANK_TCACHES 2U
22 #define WITNESS_RANK_ARENAS 3U
24 #define WITNESS_RANK_BACKGROUND_THREAD_GLOBAL 4U
26 #define WITNESS_RANK_PROF_DUMP 5U
27 #define WITNESS_RANK_PROF_BT2GCTX 6U
28 #define WITNESS_RANK_PROF_TDATAS 7U
29 #define WITNESS_RANK_PROF_TDATA 8U
30 #define WITNESS_RANK_PROF_GCTX 9U
32 #define WITNESS_RANK_BACKGROUND_THREAD 10U
35 * Used as an argument to witness_assert_depth_to_rank() in order to validate
36 * depth excluding non-core locks with lower ranks. Since the rank argument to
37 * witness_assert_depth_to_rank() is inclusive rather than exclusive, this
38 * definition can have the same value as the minimally ranked core lock.
40 #define WITNESS_RANK_CORE 11U
42 #define WITNESS_RANK_DECAY 11U
43 #define WITNESS_RANK_TCACHE_QL 12U
44 #define WITNESS_RANK_EXTENT_GROW 13U
45 #define WITNESS_RANK_EXTENTS 14U
46 #define WITNESS_RANK_EXTENT_AVAIL 15U
48 #define WITNESS_RANK_EXTENT_POOL 16U
49 #define WITNESS_RANK_RTREE 17U
50 #define WITNESS_RANK_BASE 18U
51 #define WITNESS_RANK_ARENA_LARGE 19U
53 #define WITNESS_RANK_LEAF 0xffffffffU
54 #define WITNESS_RANK_BIN WITNESS_RANK_LEAF
55 #define WITNESS_RANK_ARENA_STATS WITNESS_RANK_LEAF
56 #define WITNESS_RANK_DSS WITNESS_RANK_LEAF
57 #define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF
58 #define WITNESS_RANK_PROF_ACCUM WITNESS_RANK_LEAF
59 #define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF
60 #define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF
61 #define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF
62 #define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF
64 /******************************************************************************/
65 /* PER-WITNESS DATA */
66 /******************************************************************************/
67 #if defined(JEMALLOC_DEBUG)
68 # define WITNESS_INITIALIZER(name, rank) {name, rank, NULL, NULL, {NULL, NULL}}
70 # define WITNESS_INITIALIZER(name, rank)
73 typedef struct witness_s witness_t;
74 typedef unsigned witness_rank_t;
75 typedef ql_head(witness_t) witness_list_t;
76 typedef int witness_comp_t (const witness_t *, void *, const witness_t *,
80 /* Name, used for printing lock order reversal messages. */
84 * Witness rank, where 0 is lowest and UINT_MAX is highest. Witnesses
85 * must be acquired in order of increasing rank.
90 * If two witnesses are of equal rank and they have the samp comp
91 * function pointer, it is called as a last attempt to differentiate
92 * between witnesses of equal rank.
96 /* Opaque data, passed to comp(). */
99 /* Linkage for thread's currently owned locks. */
100 ql_elm(witness_t) link;
103 /******************************************************************************/
104 /* PER-THREAD DATA */
105 /******************************************************************************/
106 typedef struct witness_tsd_s witness_tsd_t;
107 struct witness_tsd_s {
108 witness_list_t witnesses;
112 #define WITNESS_TSD_INITIALIZER { ql_head_initializer(witnesses), false }
113 #define WITNESS_TSDN_NULL ((witness_tsdn_t *)0)
115 /******************************************************************************/
116 /* (PER-THREAD) NULLABILITY HELPERS */
117 /******************************************************************************/
118 typedef struct witness_tsdn_s witness_tsdn_t;
119 struct witness_tsdn_s {
120 witness_tsd_t witness_tsd;
123 JEMALLOC_ALWAYS_INLINE witness_tsdn_t *
124 witness_tsd_tsdn(witness_tsd_t *witness_tsd) {
125 return (witness_tsdn_t *)witness_tsd;
128 JEMALLOC_ALWAYS_INLINE bool
129 witness_tsdn_null(witness_tsdn_t *witness_tsdn) {
130 return witness_tsdn == NULL;
133 JEMALLOC_ALWAYS_INLINE witness_tsd_t *
134 witness_tsdn_tsd(witness_tsdn_t *witness_tsdn) {
135 assert(!witness_tsdn_null(witness_tsdn));
136 return &witness_tsdn->witness_tsd;
139 /******************************************************************************/
141 /******************************************************************************/
142 void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
143 witness_comp_t *comp, void *opaque);
145 typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
146 extern witness_lock_error_t *JET_MUTABLE witness_lock_error;
148 typedef void (witness_owner_error_t)(const witness_t *);
149 extern witness_owner_error_t *JET_MUTABLE witness_owner_error;
151 typedef void (witness_not_owner_error_t)(const witness_t *);
152 extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error;
154 typedef void (witness_depth_error_t)(const witness_list_t *,
155 witness_rank_t rank_inclusive, unsigned depth);
156 extern witness_depth_error_t *JET_MUTABLE witness_depth_error;
158 void witnesses_cleanup(witness_tsd_t *witness_tsd);
159 void witness_prefork(witness_tsd_t *witness_tsd);
160 void witness_postfork_parent(witness_tsd_t *witness_tsd);
161 void witness_postfork_child(witness_tsd_t *witness_tsd);
163 /* Helper, not intended for direct use. */
165 witness_owner(witness_tsd_t *witness_tsd, const witness_t *witness) {
166 witness_list_t *witnesses;
169 cassert(config_debug);
171 witnesses = &witness_tsd->witnesses;
172 ql_foreach(w, witnesses, link) {
182 witness_assert_owner(witness_tsdn_t *witness_tsdn, const witness_t *witness) {
183 witness_tsd_t *witness_tsd;
189 if (witness_tsdn_null(witness_tsdn)) {
192 witness_tsd = witness_tsdn_tsd(witness_tsdn);
193 if (witness->rank == WITNESS_RANK_OMIT) {
197 if (witness_owner(witness_tsd, witness)) {
200 witness_owner_error(witness);
204 witness_assert_not_owner(witness_tsdn_t *witness_tsdn,
205 const witness_t *witness) {
206 witness_tsd_t *witness_tsd;
207 witness_list_t *witnesses;
214 if (witness_tsdn_null(witness_tsdn)) {
217 witness_tsd = witness_tsdn_tsd(witness_tsdn);
218 if (witness->rank == WITNESS_RANK_OMIT) {
222 witnesses = &witness_tsd->witnesses;
223 ql_foreach(w, witnesses, link) {
225 witness_not_owner_error(witness);
231 witness_assert_depth_to_rank(witness_tsdn_t *witness_tsdn,
232 witness_rank_t rank_inclusive, unsigned depth) {
233 witness_tsd_t *witness_tsd;
235 witness_list_t *witnesses;
242 if (witness_tsdn_null(witness_tsdn)) {
245 witness_tsd = witness_tsdn_tsd(witness_tsdn);
248 witnesses = &witness_tsd->witnesses;
249 w = ql_last(witnesses, link);
251 ql_reverse_foreach(w, witnesses, link) {
252 if (w->rank < rank_inclusive) {
259 witness_depth_error(witnesses, rank_inclusive, depth);
264 witness_assert_depth(witness_tsdn_t *witness_tsdn, unsigned depth) {
265 witness_assert_depth_to_rank(witness_tsdn, WITNESS_RANK_MIN, depth);
269 witness_assert_lockless(witness_tsdn_t *witness_tsdn) {
270 witness_assert_depth(witness_tsdn, 0);
274 witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) {
275 witness_tsd_t *witness_tsd;
276 witness_list_t *witnesses;
283 if (witness_tsdn_null(witness_tsdn)) {
286 witness_tsd = witness_tsdn_tsd(witness_tsdn);
287 if (witness->rank == WITNESS_RANK_OMIT) {
291 witness_assert_not_owner(witness_tsdn, witness);
293 witnesses = &witness_tsd->witnesses;
294 w = ql_last(witnesses, link);
296 /* No other locks; do nothing. */
297 } else if (witness_tsd->forking && w->rank <= witness->rank) {
298 /* Forking, and relaxed ranking satisfied. */
299 } else if (w->rank > witness->rank) {
300 /* Not forking, rank order reversal. */
301 witness_lock_error(witnesses, witness);
302 } else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
303 witness->comp || w->comp(w, w->opaque, witness, witness->opaque) >
306 * Missing/incompatible comparison function, or comparison
307 * function indicates rank order reversal.
309 witness_lock_error(witnesses, witness);
312 ql_elm_new(witness, link);
313 ql_tail_insert(witnesses, witness, link);
317 witness_unlock(witness_tsdn_t *witness_tsdn, witness_t *witness) {
318 witness_tsd_t *witness_tsd;
319 witness_list_t *witnesses;
325 if (witness_tsdn_null(witness_tsdn)) {
328 witness_tsd = witness_tsdn_tsd(witness_tsdn);
329 if (witness->rank == WITNESS_RANK_OMIT) {
334 * Check whether owner before removal, rather than relying on
335 * witness_assert_owner() to abort, so that unit tests can test this
336 * function's failure mode without causing undefined behavior.
338 if (witness_owner(witness_tsd, witness)) {
339 witnesses = &witness_tsd->witnesses;
340 ql_remove(witnesses, witness, link);
342 witness_assert_owner(witness_tsdn, witness);
346 #endif /* JEMALLOC_INTERNAL_WITNESS_H */