diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index c4a44e3..4626e9b 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -53,11 +53,23 @@ This manual describes jemalloc @jemalloc_version@. More information can be found at the jemalloc website. + + The following configuration options are enabled in libc's built-in + jemalloc: , + , , + , , + , , and + . Additionally, + is enabled in development versions of + FreeBSD (controlled by the MALLOC_PRODUCTION make + variable). + SYNOPSIS - #include <jemalloc/jemalloc.h> + #include <stdlib.h> +#include <malloc_np.h> Standard API @@ -2961,4 +2973,18 @@ malloc_conf = "lg_chunk:24";]]> The posix_memalign function conforms to IEEE Std 1003.1-2001 (“POSIX.1”). + + HISTORY + The malloc_usable_size and + posix_memalign functions first appeared in + FreeBSD 7.0. + + The aligned_alloc, + malloc_stats_print, and + mallctl* functions first appeared in + FreeBSD 10.0. + + The *allocx functions first appeared + in FreeBSD 11.0. + diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index b1de2b6..da6b6d2 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -718,8 +718,13 @@ arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind) JEMALLOC_ALWAYS_INLINE const arena_chunk_map_misc_t * arena_miscelm_get_const(const arena_chunk_t *chunk, size_t pageind) { +#if 1 /* Work around gcc bug. */ + arena_chunk_t *mchunk = (arena_chunk_t *)chunk; + return (arena_miscelm_get_mutable(mchunk, pageind)); +#else return (arena_miscelm_get_mutable((arena_chunk_t *)chunk, pageind)); +#endif } JEMALLOC_ALWAYS_INLINE size_t @@ -778,8 +783,13 @@ arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind) JEMALLOC_ALWAYS_INLINE const size_t * arena_mapbitsp_get_const(const arena_chunk_t *chunk, size_t pageind) { +#if 1 /* Work around gcc bug. */ + arena_chunk_t *mchunk = (arena_chunk_t *)chunk; + return (arena_mapbitsp_get_mutable(mchunk, pageind)); +#else return (arena_mapbitsp_get_mutable((arena_chunk_t *)chunk, pageind)); +#endif } JEMALLOC_ALWAYS_INLINE size_t diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 8f82edd..78e2df2 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -8,6 +8,9 @@ #include #endif +#include "un-namespace.h" +#include "libc_private.h" + #define JEMALLOC_NO_DEMANGLE #ifdef JEMALLOC_JET # define JEMALLOC_N(n) jet_##n @@ -42,13 +45,7 @@ static const bool config_fill = false #endif ; -static const bool config_lazy_lock = -#ifdef JEMALLOC_LAZY_LOCK - true -#else - false -#endif - ; +static const bool config_lazy_lock = true; static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; static const bool config_prof = #ifdef JEMALLOC_PROF diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h index 2b8ca5d..42d97f2 100644 --- a/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/include/jemalloc/internal/jemalloc_internal_decls.h @@ -1,6 +1,9 @@ #ifndef JEMALLOC_INTERNAL_DECLS_H #define JEMALLOC_INTERNAL_DECLS_H +#include "libc_private.h" +#include "namespace.h" + #include #ifdef _WIN32 # include diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h index 5221799..60ab041 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h @@ -52,9 +52,6 @@ struct malloc_mutex_s { #ifdef JEMALLOC_LAZY_LOCK extern bool isthreaded; -#else -# undef isthreaded /* Undo private_namespace.h definition. */ -# define isthreaded true #endif bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, @@ -62,6 +59,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); +bool malloc_mutex_first_thread(void); bool malloc_mutex_boot(void); #endif /* JEMALLOC_H_EXTERNS */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index f2b6a55..69369c9 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -311,7 +311,6 @@ iralloct_realign isalloc isdalloct isqalloc -isthreaded ivsalloc ixalloc jemalloc_postfork_child diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h new file mode 100644 index 0000000..c58a8f3 --- /dev/null +++ b/include/jemalloc/jemalloc_FreeBSD.h @@ -0,0 +1,162 @@ +/* + * Override settings that were generated in jemalloc_defs.h as necessary. + */ + +#undef JEMALLOC_OVERRIDE_VALLOC + +#ifndef MALLOC_PRODUCTION +#define JEMALLOC_DEBUG +#endif + +#undef JEMALLOC_DSS + +/* + * The following are architecture-dependent, so conditionally define them for + * each supported architecture. + */ +#undef JEMALLOC_TLS_MODEL +#undef STATIC_PAGE_SHIFT +#undef LG_SIZEOF_PTR +#undef LG_SIZEOF_INT +#undef LG_SIZEOF_LONG +#undef LG_SIZEOF_INTMAX_T + +#ifdef __i386__ +# define LG_SIZEOF_PTR 2 +# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) +#endif +#ifdef __ia64__ +# define LG_SIZEOF_PTR 3 +#endif +#ifdef __sparc64__ +# define LG_SIZEOF_PTR 3 +# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) +#endif +#ifdef __amd64__ +# define LG_SIZEOF_PTR 3 +# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) +#endif +#ifdef __arm__ +# define LG_SIZEOF_PTR 2 +#endif +#ifdef __aarch64__ +# define LG_SIZEOF_PTR 3 +#endif +#ifdef __mips__ +#ifdef __mips_n64 +# define LG_SIZEOF_PTR 3 +#else +# define LG_SIZEOF_PTR 2 +#endif +#endif +#ifdef __powerpc64__ +# define LG_SIZEOF_PTR 3 +#elif defined(__powerpc__) +# define LG_SIZEOF_PTR 2 +#endif +#ifdef __riscv__ +# define LG_SIZEOF_PTR 3 +#endif + +#ifndef JEMALLOC_TLS_MODEL +# define JEMALLOC_TLS_MODEL /* Default. */ +#endif + +#define STATIC_PAGE_SHIFT PAGE_SHIFT +#define LG_SIZEOF_INT 2 +#define LG_SIZEOF_LONG LG_SIZEOF_PTR +#define LG_SIZEOF_INTMAX_T 3 + +#undef CPU_SPINWAIT +#include +#include +#define CPU_SPINWAIT cpu_spinwait() + +/* Disable lazy-lock machinery, mangle isthreaded, and adjust its type. */ +#undef JEMALLOC_LAZY_LOCK +extern int __isthreaded; +#define isthreaded ((bool)__isthreaded) + +/* Mangle. */ +#undef je_malloc +#undef je_calloc +#undef je_posix_memalign +#undef je_aligned_alloc +#undef je_realloc +#undef je_free +#undef je_malloc_usable_size +#undef je_mallocx +#undef je_rallocx +#undef je_xallocx +#undef je_sallocx +#undef je_dallocx +#undef je_sdallocx +#undef je_nallocx +#undef je_mallctl +#undef je_mallctlnametomib +#undef je_mallctlbymib +#undef je_malloc_stats_print +#undef je_allocm +#undef je_rallocm +#undef je_sallocm +#undef je_dallocm +#undef je_nallocm +#define je_malloc __malloc +#define je_calloc __calloc +#define je_posix_memalign __posix_memalign +#define je_aligned_alloc __aligned_alloc +#define je_realloc __realloc +#define je_free __free +#define je_malloc_usable_size __malloc_usable_size +#define je_mallocx __mallocx +#define je_rallocx __rallocx +#define je_xallocx __xallocx +#define je_sallocx __sallocx +#define je_dallocx __dallocx +#define je_sdallocx __sdallocx +#define je_nallocx __nallocx +#define je_mallctl __mallctl +#define je_mallctlnametomib __mallctlnametomib +#define je_mallctlbymib __mallctlbymib +#define je_malloc_stats_print __malloc_stats_print +#define je_allocm __allocm +#define je_rallocm __rallocm +#define je_sallocm __sallocm +#define je_dallocm __dallocm +#define je_nallocm __nallocm +#define open _open +#define read _read +#define write _write +#define close _close +#define pthread_mutex_lock _pthread_mutex_lock +#define pthread_mutex_unlock _pthread_mutex_unlock + +#ifdef JEMALLOC_C_ +/* + * Define 'weak' symbols so that an application can have its own versions + * of malloc, calloc, realloc, free, et al. + */ +__weak_reference(__malloc, malloc); +__weak_reference(__calloc, calloc); +__weak_reference(__posix_memalign, posix_memalign); +__weak_reference(__aligned_alloc, aligned_alloc); +__weak_reference(__realloc, realloc); +__weak_reference(__free, free); +__weak_reference(__malloc_usable_size, malloc_usable_size); +__weak_reference(__mallocx, mallocx); +__weak_reference(__rallocx, rallocx); +__weak_reference(__xallocx, xallocx); +__weak_reference(__sallocx, sallocx); +__weak_reference(__dallocx, dallocx); +__weak_reference(__sdallocx, sdallocx); +__weak_reference(__nallocx, nallocx); +__weak_reference(__mallctl, mallctl); +__weak_reference(__mallctlnametomib, mallctlnametomib); +__weak_reference(__mallctlbymib, mallctlbymib); +__weak_reference(__malloc_stats_print, malloc_stats_print); +__weak_reference(__allocm, allocm); +__weak_reference(__rallocm, rallocm); +__weak_reference(__sallocm, sallocm); +__weak_reference(__dallocm, dallocm); +__weak_reference(__nallocm, nallocm); +#endif diff --git a/include/jemalloc/jemalloc_rename.sh b/include/jemalloc/jemalloc_rename.sh index f943891..47d032c 100755 --- a/include/jemalloc/jemalloc_rename.sh +++ b/include/jemalloc/jemalloc_rename.sh @@ -19,4 +19,6 @@ done cat <: */ +const char *__malloc_options_1_0 = NULL; +__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); + /* Runtime configuration options. */ const char *je_malloc_conf JEMALLOC_ATTR(weak); bool opt_abort = @@ -2673,6 +2677,107 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) */ /******************************************************************************/ /* + * Begin compatibility functions. + */ + +#define ALLOCM_LG_ALIGN(la) (la) +#define ALLOCM_ALIGN(a) (ffsl(a)-1) +#define ALLOCM_ZERO ((int)0x40) +#define ALLOCM_NO_MOVE ((int)0x80) + +#define ALLOCM_SUCCESS 0 +#define ALLOCM_ERR_OOM 1 +#define ALLOCM_ERR_NOT_MOVED 2 + +int +je_allocm(void **ptr, size_t *rsize, size_t size, int flags) +{ + void *p; + + assert(ptr != NULL); + + p = je_mallocx(size, flags); + if (p == NULL) + return (ALLOCM_ERR_OOM); + if (rsize != NULL) + *rsize = isalloc(tsdn_fetch(), p, config_prof); + *ptr = p; + return (ALLOCM_SUCCESS); +} + +int +je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) +{ + int ret; + bool no_move = flags & ALLOCM_NO_MOVE; + + assert(ptr != NULL); + assert(*ptr != NULL); + assert(size != 0); + assert(SIZE_T_MAX - size >= extra); + + if (no_move) { + size_t usize = je_xallocx(*ptr, size, extra, flags); + ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; + if (rsize != NULL) + *rsize = usize; + } else { + void *p = je_rallocx(*ptr, size+extra, flags); + if (p != NULL) { + *ptr = p; + ret = ALLOCM_SUCCESS; + } else + ret = ALLOCM_ERR_OOM; + if (rsize != NULL) + *rsize = isalloc(tsdn_fetch(), *ptr, config_prof); + } + return (ret); +} + +int +je_sallocm(const void *ptr, size_t *rsize, int flags) +{ + + assert(rsize != NULL); + *rsize = je_sallocx(ptr, flags); + return (ALLOCM_SUCCESS); +} + +int +je_dallocm(void *ptr, int flags) +{ + + je_dallocx(ptr, flags); + return (ALLOCM_SUCCESS); +} + +int +je_nallocm(size_t *rsize, size_t size, int flags) +{ + size_t usize; + + usize = je_nallocx(size, flags); + if (usize == 0) + return (ALLOCM_ERR_OOM); + if (rsize != NULL) + *rsize = usize; + return (ALLOCM_SUCCESS); +} + +#undef ALLOCM_LG_ALIGN +#undef ALLOCM_ALIGN +#undef ALLOCM_ZERO +#undef ALLOCM_NO_MOVE + +#undef ALLOCM_SUCCESS +#undef ALLOCM_ERR_OOM +#undef ALLOCM_ERR_NOT_MOVED + +/* + * End compatibility functions. + */ +/******************************************************************************/ +/* * The following functions are used by threading libraries for protection of * malloc during fork(). */ @@ -2814,4 +2919,11 @@ jemalloc_postfork_child(void) ctl_postfork_child(tsd_tsdn(tsd)); } +void +_malloc_first_thread(void) +{ + + (void)malloc_mutex_first_thread(); +} + /******************************************************************************/ diff --git a/src/mutex.c b/src/mutex.c index a1fac34..a24e420 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -66,6 +66,17 @@ pthread_create(pthread_t *__restrict thread, #ifdef JEMALLOC_MUTEX_INIT_CB JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, void *(calloc_cb)(size_t, size_t)); + +#pragma weak _pthread_mutex_init_calloc_cb +int +_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, + void *(calloc_cb)(size_t, size_t)) +{ + + return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t))) + __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex, + calloc_cb)); +} #endif bool @@ -140,7 +151,7 @@ malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex) } bool -malloc_mutex_boot(void) +malloc_mutex_first_thread(void) { #ifdef JEMALLOC_MUTEX_INIT_CB @@ -154,3 +165,14 @@ malloc_mutex_boot(void) #endif return (false); } + +bool +malloc_mutex_boot(void) +{ + +#ifndef JEMALLOC_MUTEX_INIT_CB + return (malloc_mutex_first_thread()); +#else + return (false); +#endif +} diff --git a/src/util.c b/src/util.c index a1c4a2a..04f9153 100644 --- a/src/util.c +++ b/src/util.c @@ -67,6 +67,22 @@ wrtmessage(void *cbopaque, const char *s) JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); +JEMALLOC_ATTR(visibility("hidden")) +void +wrtmessage_1_0(const char *s1, const char *s2, const char *s3, + const char *s4) +{ + + wrtmessage(NULL, s1); + wrtmessage(NULL, s2); + wrtmessage(NULL, s3); + wrtmessage(NULL, s4); +} + +void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3, + const char *s4) = wrtmessage_1_0; +__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0); + /* * Wrapper around malloc_message() that avoids the need for * je_malloc_message(...) throughout the code.