contrib/jemalloc/src/quarantine.c

   1 #define JEMALLOC_QUARANTINE_C_
   2 #include "jemalloc/internal/jemalloc_internal.h"
   3
   4 /*
   5  * quarantine pointers close to NULL are used to encode state information that
   6  * is used for cleaning up during thread shutdown.
   7  */
   8 #define QUARANTINE_STATE_REINCARNATED   ((quarantine_t *)(uintptr_t)1)
   9 #define QUARANTINE_STATE_PURGATORY      ((quarantine_t *)(uintptr_t)2)
  10 #define QUARANTINE_STATE_MAX            QUARANTINE_STATE_PURGATORY
  11
  12 /******************************************************************************/
  13 /* Data. */
  14
  15 malloc_tsd_data(, quarantine, quarantine_t *, NULL)
  16
  17 /******************************************************************************/
  18 /* Function prototypes for non-inline static functions. */
  19
  20 static quarantine_t     *quarantine_grow(quarantine_t *quarantine);
  21 static void     quarantine_drain_one(quarantine_t *quarantine);
  22 static void     quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
  23
  24 /******************************************************************************/
  25
  26 quarantine_t *
  27 quarantine_init(size_t lg_maxobjs)
  28 {
  29         quarantine_t *quarantine;
  30
  31         quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) +
  32             ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)));
  33         if (quarantine == NULL)
  34                 return (NULL);
  35         quarantine->curbytes = 0;
  36         quarantine->curobjs = 0;
  37         quarantine->first = 0;
  38         quarantine->lg_maxobjs = lg_maxobjs;
  39
  40         quarantine_tsd_set(&quarantine);
  41
  42         return (quarantine);
  43 }
  44
  45 static quarantine_t *
  46 quarantine_grow(quarantine_t *quarantine)
  47 {
  48         quarantine_t *ret;
  49
  50         ret = quarantine_init(quarantine->lg_maxobjs + 1);
  51         if (ret == NULL) {
  52                 quarantine_drain_one(quarantine);
  53                 return (quarantine);
  54         }
  55
  56         ret->curbytes = quarantine->curbytes;
  57         ret->curobjs = quarantine->curobjs;
  58         if (quarantine->first + quarantine->curobjs <= (ZU(1) <<
  59             quarantine->lg_maxobjs)) {
  60                 /* objs ring buffer data are contiguous. */
  61                 memcpy(ret->objs, &quarantine->objs[quarantine->first],
  62                     quarantine->curobjs * sizeof(quarantine_obj_t));
  63         } else {
  64                 /* objs ring buffer data wrap around. */
  65                 size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) -
  66                     quarantine->first;
  67                 size_t ncopy_b = quarantine->curobjs - ncopy_a;
  68
  69                 memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a
  70                     * sizeof(quarantine_obj_t));
  71                 memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
  72                     sizeof(quarantine_obj_t));
  73         }
  74         idalloc(quarantine);
  75
  76         return (ret);
  77 }
  78
  79 static void
  80 quarantine_drain_one(quarantine_t *quarantine)
  81 {
  82         quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
  83         assert(obj->usize == isalloc(obj->ptr, config_prof));
  84         idalloc(obj->ptr);
  85         quarantine->curbytes -= obj->usize;
  86         quarantine->curobjs--;
  87         quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
  88             quarantine->lg_maxobjs) - 1);
  89 }
  90
  91 static void
  92 quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
  93 {
  94
  95         while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
  96                 quarantine_drain_one(quarantine);
  97 }
  98
  99 void
 100 quarantine(void *ptr)
 101 {
 102         quarantine_t *quarantine;
 103         size_t usize = isalloc(ptr, config_prof);
 104
 105         cassert(config_fill);
 106         assert(opt_quarantine);
 107
 108         quarantine = *quarantine_tsd_get();
 109         if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) {
 110                 if (quarantine == QUARANTINE_STATE_PURGATORY) {
 111                         /*
 112                          * Make a note that quarantine() was called after
 113                          * quarantine_cleanup() was called.
 114                          */
 115                         quarantine = QUARANTINE_STATE_REINCARNATED;
 116                         quarantine_tsd_set(&quarantine);
 117                 }
 118                 idalloc(ptr);
 119                 return;
 120         }
 121         /*
 122          * Drain one or more objects if the quarantine size limit would be
 123          * exceeded by appending ptr.
 124          */
 125         if (quarantine->curbytes + usize > opt_quarantine) {
 126                 size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
 127                     - usize : 0;
 128                 quarantine_drain(quarantine, upper_bound);
 129         }
 130         /* Grow the quarantine ring buffer if it's full. */
 131         if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
 132                 quarantine = quarantine_grow(quarantine);
 133         /* quarantine_grow() must free a slot if it fails to grow. */
 134         assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
 135         /* Append ptr if its size doesn't exceed the quarantine size. */
 136         if (quarantine->curbytes + usize <= opt_quarantine) {
 137                 size_t offset = (quarantine->first + quarantine->curobjs) &
 138                     ((ZU(1) << quarantine->lg_maxobjs) - 1);
 139                 quarantine_obj_t *obj = &quarantine->objs[offset];
 140                 obj->ptr = ptr;
 141                 obj->usize = usize;
 142                 quarantine->curbytes += usize;
 143                 quarantine->curobjs++;
 144                 if (opt_junk)
 145                         memset(ptr, 0x5a, usize);
 146         } else {
 147                 assert(quarantine->curbytes == 0);
 148                 idalloc(ptr);
 149         }
 150 }
 151
 152 void
 153 quarantine_cleanup(void *arg)
 154 {
 155         quarantine_t *quarantine = *(quarantine_t **)arg;
 156
 157         if (quarantine == QUARANTINE_STATE_REINCARNATED) {
 158                 /*
 159                  * Another destructor deallocated memory after this destructor
 160                  * was called.  Reset quarantine to QUARANTINE_STATE_PURGATORY
 161                  * in order to receive another callback.
 162                  */
 163                 quarantine = QUARANTINE_STATE_PURGATORY;
 164                 quarantine_tsd_set(&quarantine);
 165         } else if (quarantine == QUARANTINE_STATE_PURGATORY) {
 166                 /*
 167                  * The previous time this destructor was called, we set the key
 168                  * to QUARANTINE_STATE_PURGATORY so that other destructors
 169                  * wouldn't cause re-creation of the quarantine.  This time, do
 170                  * nothing, so that the destructor will not be called again.
 171                  */
 172         } else if (quarantine != NULL) {
 173                 quarantine_drain(quarantine, 0);
 174                 idalloc(quarantine);
 175                 quarantine = QUARANTINE_STATE_PURGATORY;
 176                 quarantine_tsd_set(&quarantine);
 177         }
 178 }
 179
 180 bool
 181 quarantine_boot(void)
 182 {
 183
 184         cassert(config_fill);
 185
 186         if (quarantine_tsd_boot())
 187                 return (true);
 188
 189         return (false);
 190 }