contrib/jemalloc/src/chunk_dss.c

   1 #define JEMALLOC_CHUNK_DSS_C_
   2 #include "jemalloc/internal/jemalloc_internal.h"
   3 /******************************************************************************/
   4 /* Data. */
   5
   6 const char      *dss_prec_names[] = {
   7         "disabled",
   8         "primary",
   9         "secondary",
  10         "N/A"
  11 };
  12
  13 /* Current dss precedence default, used when creating new arenas. */
  14 static dss_prec_t       dss_prec_default = DSS_PREC_DEFAULT;
  15
  16 /*
  17  * Protects sbrk() calls.  This avoids malloc races among threads, though it
  18  * does not protect against races with threads that call sbrk() directly.
  19  */
  20 static malloc_mutex_t   dss_mtx;
  21
  22 /* Base address of the DSS. */
  23 static void             *dss_base;
  24 /* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */
  25 static void             *dss_prev;
  26 /* Current upper limit on DSS addresses. */
  27 static void             *dss_max;
  28
  29 /******************************************************************************/
  30
  31 static void *
  32 chunk_dss_sbrk(intptr_t increment)
  33 {
  34
  35 #ifdef JEMALLOC_DSS
  36         return (sbrk(increment));
  37 #else
  38         not_implemented();
  39         return (NULL);
  40 #endif
  41 }
  42
  43 dss_prec_t
  44 chunk_dss_prec_get(void)
  45 {
  46         dss_prec_t ret;
  47
  48         if (!have_dss)
  49                 return (dss_prec_disabled);
  50         malloc_mutex_lock(&dss_mtx);
  51         ret = dss_prec_default;
  52         malloc_mutex_unlock(&dss_mtx);
  53         return (ret);
  54 }
  55
  56 bool
  57 chunk_dss_prec_set(dss_prec_t dss_prec)
  58 {
  59
  60         if (!have_dss)
  61                 return (dss_prec != dss_prec_disabled);
  62         malloc_mutex_lock(&dss_mtx);
  63         dss_prec_default = dss_prec;
  64         malloc_mutex_unlock(&dss_mtx);
  65         return (false);
  66 }
  67
  68 void *
  69 chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment,
  70     bool *zero, bool *commit)
  71 {
  72         void *ret;
  73
  74         cassert(have_dss);
  75         assert(size > 0 && (size & chunksize_mask) == 0);
  76         assert(alignment > 0 && (alignment & chunksize_mask) == 0);
  77
  78         /*
  79          * sbrk() uses a signed increment argument, so take care not to
  80          * interpret a huge allocation request as a negative increment.
  81          */
  82         if ((intptr_t)size < 0)
  83                 return (NULL);
  84
  85         malloc_mutex_lock(&dss_mtx);
  86         if (dss_prev != (void *)-1) {
  87                 size_t gap_size, cpad_size;
  88                 void *cpad, *dss_next;
  89                 intptr_t incr;
  90
  91                 /*
  92                  * The loop is necessary to recover from races with other
  93                  * threads that are using the DSS for something other than
  94                  * malloc.
  95                  */
  96                 do {
  97                         /* Avoid an unnecessary system call. */
  98                         if (new_addr != NULL && dss_max != new_addr)
  99                                 break;
 100
 101                         /* Get the current end of the DSS. */
 102                         dss_max = chunk_dss_sbrk(0);
 103
 104                         /* Make sure the earlier condition still holds. */
 105                         if (new_addr != NULL && dss_max != new_addr)
 106                                 break;
 107
 108                         /*
 109                          * Calculate how much padding is necessary to
 110                          * chunk-align the end of the DSS.
 111                          */
 112                         gap_size = (chunksize - CHUNK_ADDR2OFFSET(dss_max)) &
 113                             chunksize_mask;
 114                         /*
 115                          * Compute how much chunk-aligned pad space (if any) is
 116                          * necessary to satisfy alignment.  This space can be
 117                          * recycled for later use.
 118                          */
 119                         cpad = (void *)((uintptr_t)dss_max + gap_size);
 120                         ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max,
 121                             alignment);
 122                         cpad_size = (uintptr_t)ret - (uintptr_t)cpad;
 123                         dss_next = (void *)((uintptr_t)ret + size);
 124                         if ((uintptr_t)ret < (uintptr_t)dss_max ||
 125                             (uintptr_t)dss_next < (uintptr_t)dss_max) {
 126                                 /* Wrap-around. */
 127                                 malloc_mutex_unlock(&dss_mtx);
 128                                 return (NULL);
 129                         }
 130                         incr = gap_size + cpad_size + size;
 131                         dss_prev = chunk_dss_sbrk(incr);
 132                         if (dss_prev == dss_max) {
 133                                 /* Success. */
 134                                 dss_max = dss_next;
 135                                 malloc_mutex_unlock(&dss_mtx);
 136                                 if (cpad_size != 0) {
 137                                         chunk_hooks_t chunk_hooks =
 138                                             CHUNK_HOOKS_INITIALIZER;
 139                                         chunk_dalloc_wrapper(arena,
 140                                             &chunk_hooks, cpad, cpad_size,
 141                                             true);
 142                                 }
 143                                 if (*zero) {
 144                                         JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(
 145                                             ret, size);
 146                                         memset(ret, 0, size);
 147                                 }
 148                                 if (!*commit)
 149                                         *commit = pages_decommit(ret, size);
 150                                 return (ret);
 151                         }
 152                 } while (dss_prev != (void *)-1);
 153         }
 154         malloc_mutex_unlock(&dss_mtx);
 155
 156         return (NULL);
 157 }
 158
 159 bool
 160 chunk_in_dss(void *chunk)
 161 {
 162         bool ret;
 163
 164         cassert(have_dss);
 165
 166         malloc_mutex_lock(&dss_mtx);
 167         if ((uintptr_t)chunk >= (uintptr_t)dss_base
 168             && (uintptr_t)chunk < (uintptr_t)dss_max)
 169                 ret = true;
 170         else
 171                 ret = false;
 172         malloc_mutex_unlock(&dss_mtx);
 173
 174         return (ret);
 175 }
 176
 177 bool
 178 chunk_dss_boot(void)
 179 {
 180
 181         cassert(have_dss);
 182
 183         if (malloc_mutex_init(&dss_mtx))
 184                 return (true);
 185         dss_base = chunk_dss_sbrk(0);
 186         dss_prev = dss_base;
 187         dss_max = dss_base;
 188
 189         return (false);
 190 }
 191
 192 void
 193 chunk_dss_prefork(void)
 194 {
 195
 196         if (have_dss)
 197                 malloc_mutex_prefork(&dss_mtx);
 198 }
 199
 200 void
 201 chunk_dss_postfork_parent(void)
 202 {
 203
 204         if (have_dss)
 205                 malloc_mutex_postfork_parent(&dss_mtx);
 206 }
 207
 208 void
 209 chunk_dss_postfork_child(void)
 210 {
 211
 212         if (have_dss)
 213                 malloc_mutex_postfork_child(&dss_mtx);
 214 }
 215
 216 /******************************************************************************/