contrib/jemalloc/src/background_thread.c

   1 #define JEMALLOC_BACKGROUND_THREAD_C_
   2 #include "jemalloc/internal/jemalloc_preamble.h"
   3 #include "jemalloc/internal/jemalloc_internal_includes.h"
   4
   5 #include "jemalloc/internal/assert.h"
   6
   7 /******************************************************************************/
   8 /* Data. */
   9
  10 /* This option should be opt-in only. */
  11 #define BACKGROUND_THREAD_DEFAULT false
  12 /* Read-only after initialization. */
  13 bool opt_background_thread = BACKGROUND_THREAD_DEFAULT;
  14
  15 /* Used for thread creation, termination and stats. */
  16 malloc_mutex_t background_thread_lock;
  17 /* Indicates global state.  Atomic because decay reads this w/o locking. */
  18 atomic_b_t background_thread_enabled_state;
  19 size_t n_background_threads;
  20 /* Thread info per-index. */
  21 background_thread_info_t *background_thread_info;
  22
  23 /* False if no necessary runtime support. */
  24 bool can_enable_background_thread;
  25
  26 /******************************************************************************/
  27
  28 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
  29 #include <dlfcn.h>
  30
  31 static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *,
  32     void *(*)(void *), void *__restrict);
  33 static pthread_once_t once_control = PTHREAD_ONCE_INIT;
  34
  35 static void
  36 pthread_create_wrapper_once(void) {
  37 #ifdef JEMALLOC_LAZY_LOCK
  38         isthreaded = true;
  39 #endif
  40 }
  41
  42 int
  43 pthread_create_wrapper(pthread_t *__restrict thread, const pthread_attr_t *attr,
  44     void *(*start_routine)(void *), void *__restrict arg) {
  45         pthread_once(&once_control, pthread_create_wrapper_once);
  46
  47         return pthread_create_fptr(thread, attr, start_routine, arg);
  48 }
  49 #endif /* JEMALLOC_PTHREAD_CREATE_WRAPPER */
  50
  51 #ifndef JEMALLOC_BACKGROUND_THREAD
  52 #define NOT_REACHED { not_reached(); }
  53 bool background_thread_create(tsd_t *tsd, unsigned arena_ind) NOT_REACHED
  54 bool background_threads_enable(tsd_t *tsd) NOT_REACHED
  55 bool background_threads_disable(tsd_t *tsd) NOT_REACHED
  56 void background_thread_interval_check(tsdn_t *tsdn, arena_t *arena,
  57     arena_decay_t *decay, size_t npages_new) NOT_REACHED
  58 void background_thread_prefork0(tsdn_t *tsdn) NOT_REACHED
  59 void background_thread_prefork1(tsdn_t *tsdn) NOT_REACHED
  60 void background_thread_postfork_parent(tsdn_t *tsdn) NOT_REACHED
  61 void background_thread_postfork_child(tsdn_t *tsdn) NOT_REACHED
  62 bool background_thread_stats_read(tsdn_t *tsdn,
  63     background_thread_stats_t *stats) NOT_REACHED
  64 void background_thread_ctl_init(tsdn_t *tsdn) NOT_REACHED
  65 #undef NOT_REACHED
  66 #else
  67
  68 static bool background_thread_enabled_at_fork;
  69
  70 static void
  71 background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) {
  72         background_thread_wakeup_time_set(tsdn, info, 0);
  73         info->npages_to_purge_new = 0;
  74         if (config_stats) {
  75                 info->tot_n_runs = 0;
  76                 nstime_init(&info->tot_sleep_time, 0);
  77         }
  78 }
  79
  80 static inline bool
  81 set_current_thread_affinity(UNUSED int cpu) {
  82 #if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
  83         cpu_set_t cpuset;
  84         CPU_ZERO(&cpuset);
  85         CPU_SET(cpu, &cpuset);
  86         int ret = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
  87
  88         return (ret != 0);
  89 #else
  90         return false;
  91 #endif
  92 }
  93
  94 /* Threshold for determining when to wake up the background thread. */
  95 #define BACKGROUND_THREAD_NPAGES_THRESHOLD UINT64_C(1024)
  96 #define BILLION UINT64_C(1000000000)
  97 /* Minimal sleep interval 100 ms. */
  98 #define BACKGROUND_THREAD_MIN_INTERVAL_NS (BILLION / 10)
  99
 100 static inline size_t
 101 decay_npurge_after_interval(arena_decay_t *decay, size_t interval) {
 102         size_t i;
 103         uint64_t sum = 0;
 104         for (i = 0; i < interval; i++) {
 105                 sum += decay->backlog[i] * h_steps[i];
 106         }
 107         for (; i < SMOOTHSTEP_NSTEPS; i++) {
 108                 sum += decay->backlog[i] * (h_steps[i] - h_steps[i - interval]);
 109         }
 110
 111         return (size_t)(sum >> SMOOTHSTEP_BFP);
 112 }
 113
 114 static uint64_t
 115 arena_decay_compute_purge_interval_impl(tsdn_t *tsdn, arena_decay_t *decay,
 116     extents_t *extents) {
 117         if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
 118                 /* Use minimal interval if decay is contended. */
 119                 return BACKGROUND_THREAD_MIN_INTERVAL_NS;
 120         }
 121
 122         uint64_t interval;
 123         ssize_t decay_time = atomic_load_zd(&decay->time_ms, ATOMIC_RELAXED);
 124         if (decay_time <= 0) {
 125                 /* Purging is eagerly done or disabled currently. */
 126                 interval = BACKGROUND_THREAD_INDEFINITE_SLEEP;
 127                 goto label_done;
 128         }
 129
 130         uint64_t decay_interval_ns = nstime_ns(&decay->interval);
 131         assert(decay_interval_ns > 0);
 132         size_t npages = extents_npages_get(extents);
 133         if (npages == 0) {
 134                 unsigned i;
 135                 for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) {
 136                         if (decay->backlog[i] > 0) {
 137                                 break;
 138                         }
 139                 }
 140                 if (i == SMOOTHSTEP_NSTEPS) {
 141                         /* No dirty pages recorded.  Sleep indefinitely. */
 142                         interval = BACKGROUND_THREAD_INDEFINITE_SLEEP;
 143                         goto label_done;
 144                 }
 145         }
 146         if (npages <= BACKGROUND_THREAD_NPAGES_THRESHOLD) {
 147                 /* Use max interval. */
 148                 interval = decay_interval_ns * SMOOTHSTEP_NSTEPS;
 149                 goto label_done;
 150         }
 151
 152         size_t lb = BACKGROUND_THREAD_MIN_INTERVAL_NS / decay_interval_ns;
 153         size_t ub = SMOOTHSTEP_NSTEPS;
 154         /* Minimal 2 intervals to ensure reaching next epoch deadline. */
 155         lb = (lb < 2) ? 2 : lb;
 156         if ((decay_interval_ns * ub <= BACKGROUND_THREAD_MIN_INTERVAL_NS) ||
 157             (lb + 2 > ub)) {
 158                 interval = BACKGROUND_THREAD_MIN_INTERVAL_NS;
 159                 goto label_done;
 160         }
 161
 162         assert(lb + 2 <= ub);
 163         size_t npurge_lb, npurge_ub;
 164         npurge_lb = decay_npurge_after_interval(decay, lb);
 165         if (npurge_lb > BACKGROUND_THREAD_NPAGES_THRESHOLD) {
 166                 interval = decay_interval_ns * lb;
 167                 goto label_done;
 168         }
 169         npurge_ub = decay_npurge_after_interval(decay, ub);
 170         if (npurge_ub < BACKGROUND_THREAD_NPAGES_THRESHOLD) {
 171                 interval = decay_interval_ns * ub;
 172                 goto label_done;
 173         }
 174
 175         unsigned n_search = 0;
 176         size_t target, npurge;
 177         while ((npurge_lb + BACKGROUND_THREAD_NPAGES_THRESHOLD < npurge_ub)
 178             && (lb + 2 < ub)) {
 179                 target = (lb + ub) / 2;
 180                 npurge = decay_npurge_after_interval(decay, target);
 181                 if (npurge > BACKGROUND_THREAD_NPAGES_THRESHOLD) {
 182                         ub = target;
 183                         npurge_ub = npurge;
 184                 } else {
 185                         lb = target;
 186                         npurge_lb = npurge;
 187                 }
 188                 assert(n_search++ < lg_floor(SMOOTHSTEP_NSTEPS) + 1);
 189         }
 190         interval = decay_interval_ns * (ub + lb) / 2;
 191 label_done:
 192         interval = (interval < BACKGROUND_THREAD_MIN_INTERVAL_NS) ?
 193             BACKGROUND_THREAD_MIN_INTERVAL_NS : interval;
 194         malloc_mutex_unlock(tsdn, &decay->mtx);
 195
 196         return interval;
 197 }
 198
 199 /* Compute purge interval for background threads. */
 200 static uint64_t
 201 arena_decay_compute_purge_interval(tsdn_t *tsdn, arena_t *arena) {
 202         uint64_t i1, i2;
 203         i1 = arena_decay_compute_purge_interval_impl(tsdn, &arena->decay_dirty,
 204             &arena->extents_dirty);
 205         if (i1 == BACKGROUND_THREAD_MIN_INTERVAL_NS) {
 206                 return i1;
 207         }
 208         i2 = arena_decay_compute_purge_interval_impl(tsdn, &arena->decay_muzzy,
 209             &arena->extents_muzzy);
 210
 211         return i1 < i2 ? i1 : i2;
 212 }
 213
 214 static void
 215 background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info,
 216     uint64_t interval) {
 217         if (config_stats) {
 218                 info->tot_n_runs++;
 219         }
 220         info->npages_to_purge_new = 0;
 221
 222         struct timeval tv;
 223         /* Specific clock required by timedwait. */
 224         gettimeofday(&tv, NULL);
 225         nstime_t before_sleep;
 226         nstime_init2(&before_sleep, tv.tv_sec, tv.tv_usec * 1000);
 227
 228         int ret;
 229         if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) {
 230                 assert(background_thread_indefinite_sleep(info));
 231                 ret = pthread_cond_wait(&info->cond, &info->mtx.lock);
 232                 assert(ret == 0);
 233         } else {
 234                 assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS &&
 235                     interval <= BACKGROUND_THREAD_INDEFINITE_SLEEP);
 236                 /* We need malloc clock (can be different from tv). */
 237                 nstime_t next_wakeup;
 238                 nstime_init(&next_wakeup, 0);
 239                 nstime_update(&next_wakeup);
 240                 nstime_iadd(&next_wakeup, interval);
 241                 assert(nstime_ns(&next_wakeup) <
 242                     BACKGROUND_THREAD_INDEFINITE_SLEEP);
 243                 background_thread_wakeup_time_set(tsdn, info,
 244                     nstime_ns(&next_wakeup));
 245
 246                 nstime_t ts_wakeup;
 247                 nstime_copy(&ts_wakeup, &before_sleep);
 248                 nstime_iadd(&ts_wakeup, interval);
 249                 struct timespec ts;
 250                 ts.tv_sec = (size_t)nstime_sec(&ts_wakeup);
 251                 ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup);
 252
 253                 assert(!background_thread_indefinite_sleep(info));
 254                 ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, &ts);
 255                 assert(ret == ETIMEDOUT || ret == 0);
 256                 background_thread_wakeup_time_set(tsdn, info,
 257                     BACKGROUND_THREAD_INDEFINITE_SLEEP);
 258         }
 259         if (config_stats) {
 260                 gettimeofday(&tv, NULL);
 261                 nstime_t after_sleep;
 262                 nstime_init2(&after_sleep, tv.tv_sec, tv.tv_usec * 1000);
 263                 if (nstime_compare(&after_sleep, &before_sleep) > 0) {
 264                         nstime_subtract(&after_sleep, &before_sleep);
 265                         nstime_add(&info->tot_sleep_time, &after_sleep);
 266                 }
 267         }
 268 }
 269
 270 static bool
 271 background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) {
 272         if (unlikely(info->state == background_thread_paused)) {
 273                 malloc_mutex_unlock(tsdn, &info->mtx);
 274                 /* Wait on global lock to update status. */
 275                 malloc_mutex_lock(tsdn, &background_thread_lock);
 276                 malloc_mutex_unlock(tsdn, &background_thread_lock);
 277                 malloc_mutex_lock(tsdn, &info->mtx);
 278                 return true;
 279         }
 280
 281         return false;
 282 }
 283
 284 static inline void
 285 background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info, unsigned ind) {
 286         uint64_t min_interval = BACKGROUND_THREAD_INDEFINITE_SLEEP;
 287         unsigned narenas = narenas_total_get();
 288
 289         for (unsigned i = ind; i < narenas; i += ncpus) {
 290                 arena_t *arena = arena_get(tsdn, i, false);
 291                 if (!arena) {
 292                         continue;
 293                 }
 294                 arena_decay(tsdn, arena, true, false);
 295                 if (min_interval == BACKGROUND_THREAD_MIN_INTERVAL_NS) {
 296                         /* Min interval will be used. */
 297                         continue;
 298                 }
 299                 uint64_t interval = arena_decay_compute_purge_interval(tsdn,
 300                     arena);
 301                 assert(interval >= BACKGROUND_THREAD_MIN_INTERVAL_NS);
 302                 if (min_interval > interval) {
 303                         min_interval = interval;
 304                 }
 305         }
 306         background_thread_sleep(tsdn, info, min_interval);
 307 }
 308
 309 static bool
 310 background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) {
 311         if (info == &background_thread_info[0]) {
 312                 malloc_mutex_assert_owner(tsd_tsdn(tsd),
 313                     &background_thread_lock);
 314         } else {
 315                 malloc_mutex_assert_not_owner(tsd_tsdn(tsd),
 316                     &background_thread_lock);
 317         }
 318
 319         pre_reentrancy(tsd, NULL);
 320         malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
 321         bool has_thread;
 322         assert(info->state != background_thread_paused);
 323         if (info->state == background_thread_started) {
 324                 has_thread = true;
 325                 info->state = background_thread_stopped;
 326                 pthread_cond_signal(&info->cond);
 327         } else {
 328                 has_thread = false;
 329         }
 330         malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
 331
 332         if (!has_thread) {
 333                 post_reentrancy(tsd);
 334                 return false;
 335         }
 336         void *ret;
 337         if (pthread_join(info->thread, &ret)) {
 338                 post_reentrancy(tsd);
 339                 return true;
 340         }
 341         assert(ret == NULL);
 342         n_background_threads--;
 343         post_reentrancy(tsd);
 344
 345         return false;
 346 }
 347
 348 static void *background_thread_entry(void *ind_arg);
 349
 350 static int
 351 background_thread_create_signals_masked(pthread_t *thread,
 352     const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) {
 353         /*
 354          * Mask signals during thread creation so that the thread inherits
 355          * an empty signal set.
 356          */
 357         sigset_t set;
 358         sigfillset(&set);
 359         sigset_t oldset;
 360         int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset);
 361         if (mask_err != 0) {
 362                 return mask_err;
 363         }
 364         int create_err = pthread_create_wrapper(thread, attr, start_routine,
 365             arg);
 366         /*
 367          * Restore the signal mask.  Failure to restore the signal mask here
 368          * changes program behavior.
 369          */
 370         int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL);
 371         if (restore_err != 0) {
 372                 malloc_printf("<jemalloc>: background thread creation "
 373                     "failed (%d), and signal mask restoration failed "
 374                     "(%d)\n", create_err, restore_err);
 375                 if (opt_abort) {
 376                         abort();
 377                 }
 378         }
 379         return create_err;
 380 }
 381
 382 static void
 383 check_background_thread_creation(tsd_t *tsd, unsigned *n_created,
 384     bool *created_threads) {
 385         if (likely(*n_created == n_background_threads)) {
 386                 return;
 387         }
 388
 389         malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_info[0].mtx);
 390 label_restart:
 391         malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
 392         for (unsigned i = 1; i < ncpus; i++) {
 393                 if (created_threads[i]) {
 394                         continue;
 395                 }
 396                 background_thread_info_t *info = &background_thread_info[i];
 397                 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
 398                 assert(info->state != background_thread_paused);
 399                 bool create = (info->state == background_thread_started);
 400                 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
 401                 if (!create) {
 402                         continue;
 403                 }
 404
 405                 /*
 406                  * To avoid deadlock with prefork handlers (which waits for the
 407                  * mutex held here), unlock before calling pthread_create().
 408                  */
 409                 malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
 410
 411                 pre_reentrancy(tsd, NULL);
 412                 int err = background_thread_create_signals_masked(&info->thread,
 413                     NULL, background_thread_entry, (void *)(uintptr_t)i);
 414                 post_reentrancy(tsd);
 415
 416                 if (err == 0) {
 417                         (*n_created)++;
 418                         created_threads[i] = true;
 419                 } else {
 420                         malloc_printf("<jemalloc>: background thread "
 421                             "creation failed (%d)\n", err);
 422                         if (opt_abort) {
 423                                 abort();
 424                         }
 425                 }
 426                 /* Restart since we unlocked. */
 427                 goto label_restart;
 428         }
 429         malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_info[0].mtx);
 430         malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
 431 }
 432
 433 static void
 434 background_thread0_work(tsd_t *tsd) {
 435         /* Thread0 is also responsible for launching / terminating threads. */
 436         VARIABLE_ARRAY(bool, created_threads, ncpus);
 437         unsigned i;
 438         for (i = 1; i < ncpus; i++) {
 439                 created_threads[i] = false;
 440         }
 441         /* Start working, and create more threads when asked. */
 442         unsigned n_created = 1;
 443         while (background_thread_info[0].state != background_thread_stopped) {
 444                 if (background_thread_pause_check(tsd_tsdn(tsd),
 445                     &background_thread_info[0])) {
 446                         continue;
 447                 }
 448                 check_background_thread_creation(tsd, &n_created,
 449                     (bool *)&created_threads);
 450                 background_work_sleep_once(tsd_tsdn(tsd),
 451                     &background_thread_info[0], 0);
 452         }
 453
 454         /*
 455          * Shut down other threads at exit.  Note that the ctl thread is holding
 456          * the global background_thread mutex (and is waiting) for us.
 457          */
 458         assert(!background_thread_enabled());
 459         for (i = 1; i < ncpus; i++) {
 460                 background_thread_info_t *info = &background_thread_info[i];
 461                 assert(info->state != background_thread_paused);
 462                 if (created_threads[i]) {
 463                         background_threads_disable_single(tsd, info);
 464                 } else {
 465                         malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
 466                         /* Clear in case the thread wasn't created. */
 467                         info->state = background_thread_stopped;
 468                         malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
 469                 }
 470         }
 471         background_thread_info[0].state = background_thread_stopped;
 472         assert(n_background_threads == 1);
 473 }
 474
 475 static void
 476 background_work(tsd_t *tsd, unsigned ind) {
 477         background_thread_info_t *info = &background_thread_info[ind];
 478
 479         malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
 480         background_thread_wakeup_time_set(tsd_tsdn(tsd), info,
 481             BACKGROUND_THREAD_INDEFINITE_SLEEP);
 482         if (ind == 0) {
 483                 background_thread0_work(tsd);
 484         } else {
 485                 while (info->state != background_thread_stopped) {
 486                         if (background_thread_pause_check(tsd_tsdn(tsd),
 487                             info)) {
 488                                 continue;
 489                         }
 490                         background_work_sleep_once(tsd_tsdn(tsd), info, ind);
 491                 }
 492         }
 493         assert(info->state == background_thread_stopped);
 494         background_thread_wakeup_time_set(tsd_tsdn(tsd), info, 0);
 495         malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
 496 }
 497
 498 static void *
 499 background_thread_entry(void *ind_arg) {
 500         unsigned thread_ind = (unsigned)(uintptr_t)ind_arg;
 501         assert(thread_ind < ncpus);
 502 #ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP
 503         pthread_setname_np(pthread_self(), "jemalloc_bg_thd");
 504 #endif
 505         if (opt_percpu_arena != percpu_arena_disabled) {
 506                 set_current_thread_affinity((int)thread_ind);
 507         }
 508         /*
 509          * Start periodic background work.  We use internal tsd which avoids
 510          * side effects, for example triggering new arena creation (which in
 511          * turn triggers another background thread creation).
 512          */
 513         background_work(tsd_internal_fetch(), thread_ind);
 514         assert(pthread_equal(pthread_self(),
 515             background_thread_info[thread_ind].thread));
 516
 517         return NULL;
 518 }
 519
 520 static void
 521 background_thread_init(tsd_t *tsd, background_thread_info_t *info) {
 522         malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
 523         info->state = background_thread_started;
 524         background_thread_info_init(tsd_tsdn(tsd), info);
 525         n_background_threads++;
 526 }
 527
 528 /* Create a new background thread if needed. */
 529 bool
 530 background_thread_create(tsd_t *tsd, unsigned arena_ind) {
 531         assert(have_background_thread);
 532         malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
 533
 534         /* We create at most NCPUs threads. */
 535         size_t thread_ind = arena_ind % ncpus;
 536         background_thread_info_t *info = &background_thread_info[thread_ind];
 537
 538         bool need_new_thread;
 539         malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
 540         need_new_thread = background_thread_enabled() &&
 541             (info->state == background_thread_stopped);
 542         if (need_new_thread) {
 543                 background_thread_init(tsd, info);
 544         }
 545         malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
 546         if (!need_new_thread) {
 547                 return false;
 548         }
 549         if (arena_ind != 0) {
 550                 /* Threads are created asynchronously by Thread 0. */
 551                 background_thread_info_t *t0 = &background_thread_info[0];
 552                 malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx);
 553                 assert(t0->state == background_thread_started);
 554                 pthread_cond_signal(&t0->cond);
 555                 malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx);
 556
 557                 return false;
 558         }
 559
 560         pre_reentrancy(tsd, NULL);
 561         /*
 562          * To avoid complications (besides reentrancy), create internal
 563          * background threads with the underlying pthread_create.
 564          */
 565         int err = background_thread_create_signals_masked(&info->thread, NULL,
 566             background_thread_entry, (void *)thread_ind);
 567         post_reentrancy(tsd);
 568
 569         if (err != 0) {
 570                 malloc_printf("<jemalloc>: arena 0 background thread creation "
 571                     "failed (%d)\n", err);
 572                 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
 573                 info->state = background_thread_stopped;
 574                 n_background_threads--;
 575                 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
 576
 577                 return true;
 578         }
 579
 580         return false;
 581 }
 582
 583 bool
 584 background_threads_enable(tsd_t *tsd) {
 585         assert(n_background_threads == 0);
 586         assert(background_thread_enabled());
 587         malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
 588
 589         VARIABLE_ARRAY(bool, marked, ncpus);
 590         unsigned i, nmarked;
 591         for (i = 0; i < ncpus; i++) {
 592                 marked[i] = false;
 593         }
 594         nmarked = 0;
 595         /* Mark the threads we need to create for thread 0. */
 596         unsigned n = narenas_total_get();
 597         for (i = 1; i < n; i++) {
 598                 if (marked[i % ncpus] ||
 599                     arena_get(tsd_tsdn(tsd), i, false) == NULL) {
 600                         continue;
 601                 }
 602                 background_thread_info_t *info = &background_thread_info[i];
 603                 malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
 604                 assert(info->state == background_thread_stopped);
 605                 background_thread_init(tsd, info);
 606                 malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
 607                 marked[i % ncpus] = true;
 608                 if (++nmarked == ncpus) {
 609                         break;
 610                 }
 611         }
 612
 613         return background_thread_create(tsd, 0);
 614 }
 615
 616 bool
 617 background_threads_disable(tsd_t *tsd) {
 618         assert(!background_thread_enabled());
 619         malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock);
 620
 621         /* Thread 0 will be responsible for terminating other threads. */
 622         if (background_threads_disable_single(tsd,
 623             &background_thread_info[0])) {
 624                 return true;
 625         }
 626         assert(n_background_threads == 0);
 627
 628         return false;
 629 }
 630
 631 /* Check if we need to signal the background thread early. */
 632 void
 633 background_thread_interval_check(tsdn_t *tsdn, arena_t *arena,
 634     arena_decay_t *decay, size_t npages_new) {
 635         background_thread_info_t *info = arena_background_thread_info_get(
 636             arena);
 637         if (malloc_mutex_trylock(tsdn, &info->mtx)) {
 638                 /*
 639                  * Background thread may hold the mutex for a long period of
 640                  * time.  We'd like to avoid the variance on application
 641                  * threads.  So keep this non-blocking, and leave the work to a
 642                  * future epoch.
 643                  */
 644                 return;
 645         }
 646
 647         if (info->state != background_thread_started) {
 648                 goto label_done;
 649         }
 650         if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
 651                 goto label_done;
 652         }
 653
 654         ssize_t decay_time = atomic_load_zd(&decay->time_ms, ATOMIC_RELAXED);
 655         if (decay_time <= 0) {
 656                 /* Purging is eagerly done or disabled currently. */
 657                 goto label_done_unlock2;
 658         }
 659         uint64_t decay_interval_ns = nstime_ns(&decay->interval);
 660         assert(decay_interval_ns > 0);
 661
 662         nstime_t diff;
 663         nstime_init(&diff, background_thread_wakeup_time_get(info));
 664         if (nstime_compare(&diff, &decay->epoch) <= 0) {
 665                 goto label_done_unlock2;
 666         }
 667         nstime_subtract(&diff, &decay->epoch);
 668         if (nstime_ns(&diff) < BACKGROUND_THREAD_MIN_INTERVAL_NS) {
 669                 goto label_done_unlock2;
 670         }
 671
 672         if (npages_new > 0) {
 673                 size_t n_epoch = (size_t)(nstime_ns(&diff) / decay_interval_ns);
 674                 /*
 675                  * Compute how many new pages we would need to purge by the next
 676                  * wakeup, which is used to determine if we should signal the
 677                  * background thread.
 678                  */
 679                 uint64_t npurge_new;
 680                 if (n_epoch >= SMOOTHSTEP_NSTEPS) {
 681                         npurge_new = npages_new;
 682                 } else {
 683                         uint64_t h_steps_max = h_steps[SMOOTHSTEP_NSTEPS - 1];
 684                         assert(h_steps_max >=
 685                             h_steps[SMOOTHSTEP_NSTEPS - 1 - n_epoch]);
 686                         npurge_new = npages_new * (h_steps_max -
 687                             h_steps[SMOOTHSTEP_NSTEPS - 1 - n_epoch]);
 688                         npurge_new >>= SMOOTHSTEP_BFP;
 689                 }
 690                 info->npages_to_purge_new += npurge_new;
 691         }
 692
 693         bool should_signal;
 694         if (info->npages_to_purge_new > BACKGROUND_THREAD_NPAGES_THRESHOLD) {
 695                 should_signal = true;
 696         } else if (unlikely(background_thread_indefinite_sleep(info)) &&
 697             (extents_npages_get(&arena->extents_dirty) > 0 ||
 698             extents_npages_get(&arena->extents_muzzy) > 0 ||
 699             info->npages_to_purge_new > 0)) {
 700                 should_signal = true;
 701         } else {
 702                 should_signal = false;
 703         }
 704
 705         if (should_signal) {
 706                 info->npages_to_purge_new = 0;
 707                 pthread_cond_signal(&info->cond);
 708         }
 709 label_done_unlock2:
 710         malloc_mutex_unlock(tsdn, &decay->mtx);
 711 label_done:
 712         malloc_mutex_unlock(tsdn, &info->mtx);
 713 }
 714
 715 void
 716 background_thread_prefork0(tsdn_t *tsdn) {
 717         malloc_mutex_prefork(tsdn, &background_thread_lock);
 718         background_thread_enabled_at_fork = background_thread_enabled();
 719 }
 720
 721 void
 722 background_thread_prefork1(tsdn_t *tsdn) {
 723         for (unsigned i = 0; i < ncpus; i++) {
 724                 malloc_mutex_prefork(tsdn, &background_thread_info[i].mtx);
 725         }
 726 }
 727
 728 void
 729 background_thread_postfork_parent(tsdn_t *tsdn) {
 730         for (unsigned i = 0; i < ncpus; i++) {
 731                 malloc_mutex_postfork_parent(tsdn,
 732                     &background_thread_info[i].mtx);
 733         }
 734         malloc_mutex_postfork_parent(tsdn, &background_thread_lock);
 735 }
 736
 737 void
 738 background_thread_postfork_child(tsdn_t *tsdn) {
 739         for (unsigned i = 0; i < ncpus; i++) {
 740                 malloc_mutex_postfork_child(tsdn,
 741                     &background_thread_info[i].mtx);
 742         }
 743         malloc_mutex_postfork_child(tsdn, &background_thread_lock);
 744         if (!background_thread_enabled_at_fork) {
 745                 return;
 746         }
 747
 748         /* Clear background_thread state (reset to disabled for child). */
 749         malloc_mutex_lock(tsdn, &background_thread_lock);
 750         n_background_threads = 0;
 751         background_thread_enabled_set(tsdn, false);
 752         for (unsigned i = 0; i < ncpus; i++) {
 753                 background_thread_info_t *info = &background_thread_info[i];
 754                 malloc_mutex_lock(tsdn, &info->mtx);
 755                 info->state = background_thread_stopped;
 756                 int ret = pthread_cond_init(&info->cond, NULL);
 757                 assert(ret == 0);
 758                 background_thread_info_init(tsdn, info);
 759                 malloc_mutex_unlock(tsdn, &info->mtx);
 760         }
 761         malloc_mutex_unlock(tsdn, &background_thread_lock);
 762 }
 763
 764 bool
 765 background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) {
 766         assert(config_stats);
 767         malloc_mutex_lock(tsdn, &background_thread_lock);
 768         if (!background_thread_enabled()) {
 769                 malloc_mutex_unlock(tsdn, &background_thread_lock);
 770                 return true;
 771         }
 772
 773         stats->num_threads = n_background_threads;
 774         uint64_t num_runs = 0;
 775         nstime_init(&stats->run_interval, 0);
 776         for (unsigned i = 0; i < ncpus; i++) {
 777                 background_thread_info_t *info = &background_thread_info[i];
 778                 malloc_mutex_lock(tsdn, &info->mtx);
 779                 if (info->state != background_thread_stopped) {
 780                         num_runs += info->tot_n_runs;
 781                         nstime_add(&stats->run_interval, &info->tot_sleep_time);
 782                 }
 783                 malloc_mutex_unlock(tsdn, &info->mtx);
 784         }
 785         stats->num_runs = num_runs;
 786         if (num_runs > 0) {
 787                 nstime_idivide(&stats->run_interval, num_runs);
 788         }
 789         malloc_mutex_unlock(tsdn, &background_thread_lock);
 790
 791         return false;
 792 }
 793
 794 #undef BACKGROUND_THREAD_NPAGES_THRESHOLD
 795 #undef BILLION
 796 #undef BACKGROUND_THREAD_MIN_INTERVAL_NS
 797
 798 /*
 799  * When lazy lock is enabled, we need to make sure setting isthreaded before
 800  * taking any background_thread locks.  This is called early in ctl (instead of
 801  * wait for the pthread_create calls to trigger) because the mutex is required
 802  * before creating background threads.
 803  */
 804 void
 805 background_thread_ctl_init(tsdn_t *tsdn) {
 806         malloc_mutex_assert_not_owner(tsdn, &background_thread_lock);
 807 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
 808         pthread_once(&once_control, pthread_create_wrapper_once);
 809 #endif
 810 }
 811
 812 #endif /* defined(JEMALLOC_BACKGROUND_THREAD) */
 813
 814 bool
 815 background_thread_boot0(void) {
 816         if (!have_background_thread && opt_background_thread) {
 817                 malloc_printf("<jemalloc>: option background_thread currently "
 818                     "supports pthread only\n");
 819                 return true;
 820         }
 821
 822 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
 823         pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create");
 824         if (pthread_create_fptr == NULL) {
 825                 can_enable_background_thread = false;
 826                 if (config_lazy_lock || opt_background_thread) {
 827                         malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, "
 828                             "\"pthread_create\")\n");
 829                         abort();
 830                 }
 831         } else {
 832                 can_enable_background_thread = true;
 833         }
 834 #endif
 835         return false;
 836 }
 837
 838 bool
 839 background_thread_boot1(tsdn_t *tsdn) {
 840 #ifdef JEMALLOC_BACKGROUND_THREAD
 841         assert(have_background_thread);
 842         assert(narenas_total_get() > 0);
 843
 844         background_thread_enabled_set(tsdn, opt_background_thread);
 845         if (malloc_mutex_init(&background_thread_lock,
 846             "background_thread_global",
 847             WITNESS_RANK_BACKGROUND_THREAD_GLOBAL,
 848             malloc_mutex_rank_exclusive)) {
 849                 return true;
 850         }
 851         if (opt_background_thread) {
 852                 background_thread_ctl_init(tsdn);
 853         }
 854
 855         background_thread_info = (background_thread_info_t *)base_alloc(tsdn,
 856             b0get(), ncpus * sizeof(background_thread_info_t), CACHELINE);
 857         if (background_thread_info == NULL) {
 858                 return true;
 859         }
 860
 861         for (unsigned i = 0; i < ncpus; i++) {
 862                 background_thread_info_t *info = &background_thread_info[i];
 863                 /* Thread mutex is rank_inclusive because of thread0. */
 864                 if (malloc_mutex_init(&info->mtx, "background_thread",
 865                     WITNESS_RANK_BACKGROUND_THREAD,
 866                     malloc_mutex_address_ordered)) {
 867                         return true;
 868                 }
 869                 if (pthread_cond_init(&info->cond, NULL)) {
 870                         return true;
 871                 }
 872                 malloc_mutex_lock(tsdn, &info->mtx);
 873                 info->state = background_thread_stopped;
 874                 background_thread_info_init(tsdn, info);
 875                 malloc_mutex_unlock(tsdn, &info->mtx);
 876         }
 877 #endif
 878
 879         return false;
 880 }