1 diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
2 index 1e12fd3a..c42a7e10 100644
3 --- a/doc/jemalloc.xml.in
4 +++ b/doc/jemalloc.xml.in
6 <para>This manual describes jemalloc @jemalloc_version@. More information
7 can be found at the <ulink
8 url="http://jemalloc.net/">jemalloc website</ulink>.</para>
10 + <para>The following configuration options are enabled in libc's built-in
11 + jemalloc: <option>--enable-fill</option>,
12 + <option>--enable-lazy-lock</option>, <option>--enable-stats</option>,
13 + <option>--enable-utrace</option>, <option>--enable-xmalloc</option>, and
14 + <option>--with-malloc-conf=abort_conf:false</option>.
15 + Additionally, <option>--enable-debug</option> is enabled in development
16 + versions of FreeBSD (controlled by the
17 + <constant>MALLOC_PRODUCTION</constant> make variable).</para>
21 <title>SYNOPSIS</title>
23 - <funcsynopsisinfo>#include <<filename class="headerfile">jemalloc/jemalloc.h</filename>></funcsynopsisinfo>
24 + <funcsynopsisinfo>#include <<filename class="headerfile">stdlib.h</filename>>
25 +#include <<filename class="headerfile">malloc_np.h</filename>></funcsynopsisinfo>
27 <title>Standard API</title>
29 @@ -3376,4 +3387,18 @@ malloc_conf = "narenas:1";]]></programlisting></para>
30 <para>The <function>posix_memalign()</function> function conforms
31 to IEEE Std 1003.1-2001 (<quote>POSIX.1</quote>).</para>
33 + <refsect1 id="history">
34 + <title>HISTORY</title>
35 + <para>The <function>malloc_usable_size()</function> and
36 + <function>posix_memalign()</function> functions first appeared in FreeBSD
39 + <para>The <function>aligned_alloc()</function>,
40 + <function>malloc_stats_print()</function>, and
41 + <function>mallctl*()</function> functions first appeared in FreeBSD
44 + <para>The <function>*allocx()</function> functions first appeared in FreeBSD
48 diff --git a/include/jemalloc/internal/hooks.h b/include/jemalloc/internal/hooks.h
49 index cd49afcb..85e2a991 100644
50 --- a/include/jemalloc/internal/hooks.h
51 +++ b/include/jemalloc/internal/hooks.h
52 @@ -6,13 +6,6 @@ extern JEMALLOC_EXPORT void (*hooks_libc_hook)();
54 #define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn)
56 -#define open JEMALLOC_HOOK(open, hooks_libc_hook)
57 -#define read JEMALLOC_HOOK(read, hooks_libc_hook)
58 -#define write JEMALLOC_HOOK(write, hooks_libc_hook)
59 -#define readlink JEMALLOC_HOOK(readlink, hooks_libc_hook)
60 -#define close JEMALLOC_HOOK(close, hooks_libc_hook)
61 -#define creat JEMALLOC_HOOK(creat, hooks_libc_hook)
62 -#define secure_getenv JEMALLOC_HOOK(secure_getenv, hooks_libc_hook)
63 /* Note that this is undef'd and re-define'd in src/prof.c. */
64 #define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook)
66 diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h
67 index be70df51..84cd70da 100644
68 --- a/include/jemalloc/internal/jemalloc_internal_decls.h
69 +++ b/include/jemalloc/internal/jemalloc_internal_decls.h
71 #ifndef JEMALLOC_INTERNAL_DECLS_H
72 #define JEMALLOC_INTERNAL_DECLS_H
74 +#include "libc_private.h"
75 +#include "namespace.h"
80 diff --git a/include/jemalloc/internal/jemalloc_preamble.h.in b/include/jemalloc/internal/jemalloc_preamble.h.in
81 index e621fbc8..dbdd5d6b 100644
82 --- a/include/jemalloc/internal/jemalloc_preamble.h.in
83 +++ b/include/jemalloc/internal/jemalloc_preamble.h.in
85 #include <sys/ktrace.h>
88 +#include "un-namespace.h"
89 +#include "libc_private.h"
91 #define JEMALLOC_NO_DEMANGLE
93 # undef JEMALLOC_IS_MALLOC
94 @@ -79,13 +82,7 @@ static const bool config_fill =
98 -static const bool config_lazy_lock =
99 -#ifdef JEMALLOC_LAZY_LOCK
105 +static const bool config_lazy_lock = true;
106 static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF;
107 static const bool config_prof =
109 diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h
110 index 6520c251..0013cbe9 100644
111 --- a/include/jemalloc/internal/mutex.h
112 +++ b/include/jemalloc/internal/mutex.h
113 @@ -121,9 +121,6 @@ struct malloc_mutex_s {
115 #ifdef JEMALLOC_LAZY_LOCK
116 extern bool isthreaded;
118 -# undef isthreaded /* Undo private_namespace.h definition. */
119 -# define isthreaded true
122 bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
123 @@ -131,6 +128,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
124 void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
125 void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
126 void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
127 +bool malloc_mutex_first_thread(void);
128 bool malloc_mutex_boot(void);
129 void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex);
131 diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h
132 index 0b9841aa..f03eee61 100644
133 --- a/include/jemalloc/internal/tsd.h
134 +++ b/include/jemalloc/internal/tsd.h
135 @@ -122,7 +122,8 @@ struct tsd_s {
136 t use_a_getter_or_setter_instead_##n;
140 +/* AddressSanitizer requires TLS data to be aligned to at least 8 bytes. */
141 +} JEMALLOC_ALIGNED(16);
144 * Wrapper around tsd_t that makes it possible to avoid implicit conversion
145 diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h
147 index 00000000..b752b0e7
149 +++ b/include/jemalloc/jemalloc_FreeBSD.h
152 + * Override settings that were generated in jemalloc_defs.h as necessary.
155 +#undef JEMALLOC_OVERRIDE_VALLOC
157 +#ifndef MALLOC_PRODUCTION
158 +#define JEMALLOC_DEBUG
163 +#undef JEMALLOC_BACKGROUND_THREAD
166 + * The following are architecture-dependent, so conditionally define them for
167 + * each supported architecture.
169 +#undef JEMALLOC_TLS_MODEL
172 +#undef LG_SIZEOF_PTR
173 +#undef LG_SIZEOF_INT
174 +#undef LG_SIZEOF_LONG
175 +#undef LG_SIZEOF_INTMAX_T
178 +# define LG_VADDR 32
179 +# define LG_SIZEOF_PTR 2
180 +# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
183 +# define LG_VADDR 64
184 +# define LG_SIZEOF_PTR 3
187 +# define LG_VADDR 64
188 +# define LG_SIZEOF_PTR 3
189 +# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
192 +# define LG_VADDR 48
193 +# define LG_SIZEOF_PTR 3
194 +# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
197 +# define LG_VADDR 32
198 +# define LG_SIZEOF_PTR 2
201 +# define LG_VADDR 48
202 +# define LG_SIZEOF_PTR 3
206 +# define LG_VADDR 64
207 +# define LG_SIZEOF_PTR 3
209 +# define LG_VADDR 32
210 +# define LG_SIZEOF_PTR 2
213 +#ifdef __powerpc64__
214 +# define LG_VADDR 64
215 +# define LG_SIZEOF_PTR 3
216 +#elif defined(__powerpc__)
217 +# define LG_VADDR 32
218 +# define LG_SIZEOF_PTR 2
221 +# define LG_VADDR 64
222 +# define LG_SIZEOF_PTR 3
225 +#ifndef JEMALLOC_TLS_MODEL
226 +# define JEMALLOC_TLS_MODEL /* Default. */
229 +#define LG_PAGE PAGE_SHIFT
230 +#define LG_SIZEOF_INT 2
231 +#define LG_SIZEOF_LONG LG_SIZEOF_PTR
232 +#define LG_SIZEOF_INTMAX_T 3
235 +#include <machine/cpu.h>
236 +#include <machine/cpufunc.h>
237 +#define CPU_SPINWAIT cpu_spinwait()
239 +/* Disable lazy-lock machinery, mangle isthreaded, and adjust its type. */
240 +#undef JEMALLOC_LAZY_LOCK
241 +extern int __isthreaded;
242 +#define isthreaded ((bool)__isthreaded)
247 +#undef je_posix_memalign
248 +#undef je_aligned_alloc
251 +#undef je_malloc_usable_size
260 +#undef je_mallctlnametomib
261 +#undef je_mallctlbymib
262 +#undef je_malloc_stats_print
268 +#define je_malloc __malloc
269 +#define je_calloc __calloc
270 +#define je_posix_memalign __posix_memalign
271 +#define je_aligned_alloc __aligned_alloc
272 +#define je_realloc __realloc
273 +#define je_free __free
274 +#define je_malloc_usable_size __malloc_usable_size
275 +#define je_mallocx __mallocx
276 +#define je_rallocx __rallocx
277 +#define je_xallocx __xallocx
278 +#define je_sallocx __sallocx
279 +#define je_dallocx __dallocx
280 +#define je_sdallocx __sdallocx
281 +#define je_nallocx __nallocx
282 +#define je_mallctl __mallctl
283 +#define je_mallctlnametomib __mallctlnametomib
284 +#define je_mallctlbymib __mallctlbymib
285 +#define je_malloc_stats_print __malloc_stats_print
286 +#define je_allocm __allocm
287 +#define je_rallocm __rallocm
288 +#define je_sallocm __sallocm
289 +#define je_dallocm __dallocm
290 +#define je_nallocm __nallocm
293 +#define write _write
294 +#define close _close
295 +#define pthread_join _pthread_join
296 +#define pthread_once _pthread_once
297 +#define pthread_self _pthread_self
298 +#define pthread_equal _pthread_equal
299 +#define pthread_mutex_lock _pthread_mutex_lock
300 +#define pthread_mutex_trylock _pthread_mutex_trylock
301 +#define pthread_mutex_unlock _pthread_mutex_unlock
302 +#define pthread_cond_init _pthread_cond_init
303 +#define pthread_cond_wait _pthread_cond_wait
304 +#define pthread_cond_timedwait _pthread_cond_timedwait
305 +#define pthread_cond_signal _pthread_cond_signal
309 + * Define 'weak' symbols so that an application can have its own versions
310 + * of malloc, calloc, realloc, free, et al.
312 +__weak_reference(__malloc, malloc);
313 +__weak_reference(__calloc, calloc);
314 +__weak_reference(__posix_memalign, posix_memalign);
315 +__weak_reference(__aligned_alloc, aligned_alloc);
316 +__weak_reference(__realloc, realloc);
317 +__weak_reference(__free, free);
318 +__weak_reference(__malloc_usable_size, malloc_usable_size);
319 +__weak_reference(__mallocx, mallocx);
320 +__weak_reference(__rallocx, rallocx);
321 +__weak_reference(__xallocx, xallocx);
322 +__weak_reference(__sallocx, sallocx);
323 +__weak_reference(__dallocx, dallocx);
324 +__weak_reference(__sdallocx, sdallocx);
325 +__weak_reference(__nallocx, nallocx);
326 +__weak_reference(__mallctl, mallctl);
327 +__weak_reference(__mallctlnametomib, mallctlnametomib);
328 +__weak_reference(__mallctlbymib, mallctlbymib);
329 +__weak_reference(__malloc_stats_print, malloc_stats_print);
330 +__weak_reference(__allocm, allocm);
331 +__weak_reference(__rallocm, rallocm);
332 +__weak_reference(__sallocm, sallocm);
333 +__weak_reference(__dallocm, dallocm);
334 +__weak_reference(__nallocm, nallocm);
336 diff --git a/include/jemalloc/jemalloc_rename.sh b/include/jemalloc/jemalloc_rename.sh
337 index f9438912..47d032c1 100755
338 --- a/include/jemalloc/jemalloc_rename.sh
339 +++ b/include/jemalloc/jemalloc_rename.sh
340 @@ -19,4 +19,6 @@ done
345 +#include "jemalloc_FreeBSD.h"
347 diff --git a/src/jemalloc.c b/src/jemalloc.c
348 index f93c16fa..e0ad297b 100644
352 /******************************************************************************/
355 +/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
356 +const char *__malloc_options_1_0 = NULL;
357 +__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
359 /* Runtime configuration options. */
360 const char *je_malloc_conf
362 @@ -3160,6 +3164,103 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
364 /******************************************************************************/
366 + * Begin compatibility functions.
369 +#define ALLOCM_LG_ALIGN(la) (la)
370 +#define ALLOCM_ALIGN(a) (ffsl(a)-1)
371 +#define ALLOCM_ZERO ((int)0x40)
372 +#define ALLOCM_NO_MOVE ((int)0x80)
374 +#define ALLOCM_SUCCESS 0
375 +#define ALLOCM_ERR_OOM 1
376 +#define ALLOCM_ERR_NOT_MOVED 2
379 +je_allocm(void **ptr, size_t *rsize, size_t size, int flags) {
380 + assert(ptr != NULL);
382 + void *p = je_mallocx(size, flags);
384 + return (ALLOCM_ERR_OOM);
386 + if (rsize != NULL) {
387 + *rsize = isalloc(tsdn_fetch(), p);
390 + return ALLOCM_SUCCESS;
394 +je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) {
395 + assert(ptr != NULL);
396 + assert(*ptr != NULL);
398 + assert(SIZE_T_MAX - size >= extra);
401 + bool no_move = flags & ALLOCM_NO_MOVE;
404 + size_t usize = je_xallocx(*ptr, size, extra, flags);
405 + ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED;
406 + if (rsize != NULL) {
410 + void *p = je_rallocx(*ptr, size+extra, flags);
413 + ret = ALLOCM_SUCCESS;
415 + ret = ALLOCM_ERR_OOM;
417 + if (rsize != NULL) {
418 + *rsize = isalloc(tsdn_fetch(), *ptr);
425 +je_sallocm(const void *ptr, size_t *rsize, int flags) {
426 + assert(rsize != NULL);
427 + *rsize = je_sallocx(ptr, flags);
428 + return ALLOCM_SUCCESS;
432 +je_dallocm(void *ptr, int flags) {
433 + je_dallocx(ptr, flags);
434 + return ALLOCM_SUCCESS;
438 +je_nallocm(size_t *rsize, size_t size, int flags) {
439 + size_t usize = je_nallocx(size, flags);
441 + return ALLOCM_ERR_OOM;
443 + if (rsize != NULL) {
446 + return ALLOCM_SUCCESS;
449 +#undef ALLOCM_LG_ALIGN
452 +#undef ALLOCM_NO_MOVE
454 +#undef ALLOCM_SUCCESS
455 +#undef ALLOCM_ERR_OOM
456 +#undef ALLOCM_ERR_NOT_MOVED
459 + * End compatibility functions.
461 +/******************************************************************************/
463 * The following functions are used by threading libraries for protection of
464 * malloc during fork().
466 @@ -3323,4 +3424,11 @@ jemalloc_postfork_child(void) {
467 ctl_postfork_child(tsd_tsdn(tsd));
471 +_malloc_first_thread(void)
474 + (void)malloc_mutex_first_thread();
477 /******************************************************************************/
478 diff --git a/src/malloc_io.c b/src/malloc_io.c
479 index 7bdc13f9..c8802c70 100644
480 --- a/src/malloc_io.c
481 +++ b/src/malloc_io.c
482 @@ -75,6 +75,20 @@ wrtmessage(void *cbopaque, const char *s) {
484 JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s);
486 +JEMALLOC_ATTR(visibility("hidden"))
488 +wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) {
490 + wrtmessage(NULL, s1);
491 + wrtmessage(NULL, s2);
492 + wrtmessage(NULL, s3);
493 + wrtmessage(NULL, s4);
496 +void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
497 + const char *s4) = wrtmessage_1_0;
498 +__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
501 * Wrapper around malloc_message() that avoids the need for
502 * je_malloc_message(...) throughout the code.
503 diff --git a/src/mutex.c b/src/mutex.c
504 index 30222b3e..b2c36283 100644
507 @@ -41,6 +41,17 @@ pthread_create(pthread_t *__restrict thread,
508 #ifdef JEMALLOC_MUTEX_INIT_CB
509 JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
510 void *(calloc_cb)(size_t, size_t));
512 +#pragma weak _pthread_mutex_init_calloc_cb
514 +_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
515 + void *(calloc_cb)(size_t, size_t))
518 + return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t)))
519 + __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex,
525 @@ -131,6 +142,16 @@ mutex_addr_comp(const witness_t *witness1, void *mutex1,
529 +malloc_mutex_first_thread(void) {
531 +#ifndef JEMALLOC_MUTEX_INIT_CB
532 + return (malloc_mutex_first_thread());
539 malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
540 witness_rank_t rank, malloc_mutex_lock_order_t lock_order) {
541 mutex_prof_data_init(&mutex->prof_data);