contrib/jemalloc/include/jemalloc/internal/tsd_generic.h

   1 #ifdef JEMALLOC_INTERNAL_TSD_GENERIC_H
   2 #error This file should be included only once, by tsd.h.
   3 #endif
   4 #define JEMALLOC_INTERNAL_TSD_GENERIC_H
   5
   6 typedef struct tsd_init_block_s tsd_init_block_t;
   7 struct tsd_init_block_s {
   8         ql_elm(tsd_init_block_t) link;
   9         pthread_t thread;
  10         void *data;
  11 };
  12
  13 /* Defined in tsd.c, to allow the mutex headers to have tsd dependencies. */
  14 typedef struct tsd_init_head_s tsd_init_head_t;
  15
  16 typedef struct {
  17         bool initialized;
  18         tsd_t val;
  19 } tsd_wrapper_t;
  20
  21 void *tsd_init_check_recursion(tsd_init_head_t *head,
  22     tsd_init_block_t *block);
  23 void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
  24
  25 extern pthread_key_t tsd_tsd;
  26 extern tsd_init_head_t tsd_init_head;
  27 extern tsd_wrapper_t tsd_boot_wrapper;
  28 extern bool tsd_booted;
  29
  30 /* Initialization/cleanup. */
  31 JEMALLOC_ALWAYS_INLINE void
  32 tsd_cleanup_wrapper(void *arg) {
  33         tsd_wrapper_t *wrapper = (tsd_wrapper_t *)arg;
  34
  35         if (wrapper->initialized) {
  36                 wrapper->initialized = false;
  37                 tsd_cleanup(&wrapper->val);
  38                 if (wrapper->initialized) {
  39                         /* Trigger another cleanup round. */
  40                         if (pthread_setspecific(tsd_tsd, (void *)wrapper) != 0)
  41                         {
  42                                 malloc_write("<jemalloc>: Error setting TSD\n");
  43                                 if (opt_abort) {
  44                                         abort();
  45                                 }
  46                         }
  47                         return;
  48                 }
  49         }
  50         malloc_tsd_dalloc(wrapper);
  51 }
  52
  53 JEMALLOC_ALWAYS_INLINE void
  54 tsd_wrapper_set(tsd_wrapper_t *wrapper) {
  55         if (pthread_setspecific(tsd_tsd, (void *)wrapper) != 0) {
  56                 malloc_write("<jemalloc>: Error setting TSD\n");
  57                 abort();
  58         }
  59 }
  60
  61 JEMALLOC_ALWAYS_INLINE tsd_wrapper_t *
  62 tsd_wrapper_get(bool init) {
  63         tsd_wrapper_t *wrapper = (tsd_wrapper_t *)pthread_getspecific(tsd_tsd);
  64
  65         if (init && unlikely(wrapper == NULL)) {
  66                 tsd_init_block_t block;
  67                 wrapper = (tsd_wrapper_t *)
  68                     tsd_init_check_recursion(&tsd_init_head, &block);
  69                 if (wrapper) {
  70                         return wrapper;
  71                 }
  72                 wrapper = (tsd_wrapper_t *)
  73                     malloc_tsd_malloc(sizeof(tsd_wrapper_t));
  74                 block.data = (void *)wrapper;
  75                 if (wrapper == NULL) {
  76                         malloc_write("<jemalloc>: Error allocating TSD\n");
  77                         abort();
  78                 } else {
  79                         wrapper->initialized = false;
  80       JEMALLOC_DIAGNOSTIC_PUSH
  81       JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
  82                         tsd_t initializer = TSD_INITIALIZER;
  83       JEMALLOC_DIAGNOSTIC_POP
  84                         wrapper->val = initializer;
  85                 }
  86                 tsd_wrapper_set(wrapper);
  87                 tsd_init_finish(&tsd_init_head, &block);
  88         }
  89         return wrapper;
  90 }
  91
  92 JEMALLOC_ALWAYS_INLINE bool
  93 tsd_boot0(void) {
  94         if (pthread_key_create(&tsd_tsd, tsd_cleanup_wrapper) != 0) {
  95                 return true;
  96         }
  97         tsd_wrapper_set(&tsd_boot_wrapper);
  98         tsd_booted = true;
  99         return false;
 100 }
 101
 102 JEMALLOC_ALWAYS_INLINE void
 103 tsd_boot1(void) {
 104         tsd_wrapper_t *wrapper;
 105         wrapper = (tsd_wrapper_t *)malloc_tsd_malloc(sizeof(tsd_wrapper_t));
 106         if (wrapper == NULL) {
 107                 malloc_write("<jemalloc>: Error allocating TSD\n");
 108                 abort();
 109         }
 110         tsd_boot_wrapper.initialized = false;
 111         tsd_cleanup(&tsd_boot_wrapper.val);
 112         wrapper->initialized = false;
 113   JEMALLOC_DIAGNOSTIC_PUSH
 114   JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
 115         tsd_t initializer = TSD_INITIALIZER;
 116   JEMALLOC_DIAGNOSTIC_POP
 117         wrapper->val = initializer;
 118         tsd_wrapper_set(wrapper);
 119 }
 120
 121 JEMALLOC_ALWAYS_INLINE bool
 122 tsd_boot(void) {
 123         if (tsd_boot0()) {
 124                 return true;
 125         }
 126         tsd_boot1();
 127         return false;
 128 }
 129
 130 JEMALLOC_ALWAYS_INLINE bool
 131 tsd_booted_get(void) {
 132         return tsd_booted;
 133 }
 134
 135 JEMALLOC_ALWAYS_INLINE bool
 136 tsd_get_allocates(void) {
 137         return true;
 138 }
 139
 140 /* Get/set. */
 141 JEMALLOC_ALWAYS_INLINE tsd_t *
 142 tsd_get(bool init) {
 143         tsd_wrapper_t *wrapper;
 144
 145         assert(tsd_booted);
 146         wrapper = tsd_wrapper_get(init);
 147         if (tsd_get_allocates() && !init && wrapper == NULL) {
 148                 return NULL;
 149         }
 150         return &wrapper->val;
 151 }
 152
 153 JEMALLOC_ALWAYS_INLINE void
 154 tsd_set(tsd_t *val) {
 155         tsd_wrapper_t *wrapper;
 156
 157         assert(tsd_booted);
 158         wrapper = tsd_wrapper_get(true);
 159         if (likely(&wrapper->val != val)) {
 160                 wrapper->val = *(val);
 161         }
 162         wrapper->initialized = true;
 163 }