Unverified Commit c4b4b6c0 authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Merge pull request #9623 from yoav-steinberg/upgrade_jemalloc_5.2.1

Upgraded to jemalloc 5.2.1 from 5.1.0.
Cherry picked all relevant fixes (by diffing our 5.1.0 to upstream 5.10 and finding relevant commits).
Details of what was done:

[cherry-picked] fd7d51c3 2021-05-03 Resolve nonsense static analysis warnings (Oran Agra)
[cherry-picked] 448c435b 2020-09-29 Fix compilation warnings in Lua and jemalloc dependencies (#7785) (YoongHM)
[skipped - already in upstream] 9216b96b 2020-09-21 Fix compilation warning in jemalloc's malloc_vsnprintf (#7789) (YoongHM)
[cherry-picked] 88d71f47 2020-05-20 fix a rare active defrag edge case bug leading to stagnation (Oran Agra)
[skipped - already in upstream] 2fec7d9c 2019-05-30 Jemalloc: Avoid blocking on background thread lock for stats.
[cherry-picked] 920158ec 2018-07-11 Active defrag fixes for 32bit builds (again) (Oran Agra)
[cherry-picked] e8099cab 2018-06-26 add defrag hint support into jemalloc 5 (Oran Agra)
[re-done] 4e729fcd 2018-05-24 Generate configure for Jemalloc. (antirez)

Additionally had to do this:
7727cc2 2021-10-10 Fix defrag to support sharded bins in arena (added in v5.2.1) (Yoav Steinberg)

When reviewing please look at all except the first commit which is just replacing 5.1.0 with 5.2.1 sources.
Also I think we should merge this without squashing to preserve the changes we did to to jemalloc.
parents 276b460e 85737e67
#ifndef JEMALLOC_INTERNAL_HOOK_H
#define JEMALLOC_INTERNAL_HOOK_H
#include "jemalloc/internal/tsd.h"
/*
* This API is *extremely* experimental, and may get ripped out, changed in API-
* and ABI-incompatible ways, be insufficiently or incorrectly documented, etc.
*
* It allows hooking the stateful parts of the API to see changes as they
* happen.
*
* Allocation hooks are called after the allocation is done, free hooks are
* called before the free is done, and expand hooks are called after the
* allocation is expanded.
*
* For realloc and rallocx, if the expansion happens in place, the expansion
* hook is called. If it is moved, then the alloc hook is called on the new
* location, and then the free hook is called on the old location (i.e. both
* hooks are invoked in between the alloc and the dalloc).
*
* If we return NULL from OOM, then usize might not be trustworthy. Calling
* realloc(NULL, size) only calls the alloc hook, and calling realloc(ptr, 0)
* only calls the free hook. (Calling realloc(NULL, 0) is treated as malloc(0),
* and only calls the alloc hook).
*
* Reentrancy:
* Reentrancy is guarded against from within the hook implementation. If you
* call allocator functions from within a hook, the hooks will not be invoked
* again.
* Threading:
* The installation of a hook synchronizes with all its uses. If you can
* prove the installation of a hook happens-before a jemalloc entry point,
* then the hook will get invoked (unless there's a racing removal).
*
* Hook insertion appears to be atomic at a per-thread level (i.e. if a thread
* allocates and has the alloc hook invoked, then a subsequent free on the
* same thread will also have the free hook invoked).
*
* The *removal* of a hook does *not* block until all threads are done with
* the hook. Hook authors have to be resilient to this, and need some
* out-of-band mechanism for cleaning up any dynamically allocated memory
* associated with their hook.
* Ordering:
* Order of hook execution is unspecified, and may be different than insertion
* order.
*/
#define HOOK_MAX 4
enum hook_alloc_e {
hook_alloc_malloc,
hook_alloc_posix_memalign,
hook_alloc_aligned_alloc,
hook_alloc_calloc,
hook_alloc_memalign,
hook_alloc_valloc,
hook_alloc_mallocx,
/* The reallocating functions have both alloc and dalloc variants */
hook_alloc_realloc,
hook_alloc_rallocx,
};
/*
* We put the enum typedef after the enum, since this file may get included by
* jemalloc_cpp.cpp, and C++ disallows enum forward declarations.
*/
typedef enum hook_alloc_e hook_alloc_t;
enum hook_dalloc_e {
hook_dalloc_free,
hook_dalloc_dallocx,
hook_dalloc_sdallocx,
/*
* The dalloc halves of reallocation (not called if in-place expansion
* happens).
*/
hook_dalloc_realloc,
hook_dalloc_rallocx,
};
typedef enum hook_dalloc_e hook_dalloc_t;
enum hook_expand_e {
hook_expand_realloc,
hook_expand_rallocx,
hook_expand_xallocx,
};
typedef enum hook_expand_e hook_expand_t;
typedef void (*hook_alloc)(
void *extra, hook_alloc_t type, void *result, uintptr_t result_raw,
uintptr_t args_raw[3]);
typedef void (*hook_dalloc)(
void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
typedef void (*hook_expand)(
void *extra, hook_expand_t type, void *address, size_t old_usize,
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
typedef struct hooks_s hooks_t;
struct hooks_s {
hook_alloc alloc_hook;
hook_dalloc dalloc_hook;
hook_expand expand_hook;
void *extra;
};
/*
* Begin implementation details; everything above this point might one day live
* in a public API. Everything below this point never will.
*/
/*
* The realloc pathways haven't gotten any refactoring love in a while, and it's
* fairly difficult to pass information from the entry point to the hooks. We
* put the informaiton the hooks will need into a struct to encapsulate
* everything.
*
* Much of these pathways are force-inlined, so that the compiler can avoid
* materializing this struct until we hit an extern arena function. For fairly
* goofy reasons, *many* of the realloc paths hit an extern arena function.
* These paths are cold enough that it doesn't matter; eventually, we should
* rewrite the realloc code to make the expand-in-place and the
* free-then-realloc paths more orthogonal, at which point we don't need to
* spread the hook logic all over the place.
*/
typedef struct hook_ralloc_args_s hook_ralloc_args_t;
struct hook_ralloc_args_s {
/* I.e. as opposed to rallocx. */
bool is_realloc;
/*
* The expand hook takes 4 arguments, even if only 3 are actually used;
* we add an extra one in case the user decides to memcpy without
* looking too closely at the hooked function.
*/
uintptr_t args[4];
};
/*
* Returns an opaque handle to be used when removing the hook. NULL means that
* we couldn't install the hook.
*/
bool hook_boot();
void *hook_install(tsdn_t *tsdn, hooks_t *hooks);
/* Uninstalls the hook with the handle previously returned from hook_install. */
void hook_remove(tsdn_t *tsdn, void *opaque);
/* Hooks */
void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
uintptr_t args_raw[3]);
void hook_invoke_dalloc(hook_dalloc_t type, void *address,
uintptr_t args_raw[3]);
void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
#endif /* JEMALLOC_INTERNAL_HOOK_H */
#ifndef JEMALLOC_INTERNAL_HOOKS_H
#define JEMALLOC_INTERNAL_HOOKS_H
extern JEMALLOC_EXPORT void (*hooks_arena_new_hook)();
extern JEMALLOC_EXPORT void (*hooks_libc_hook)();
#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn)
#define open JEMALLOC_HOOK(open, hooks_libc_hook)
#define read JEMALLOC_HOOK(read, hooks_libc_hook)
#define write JEMALLOC_HOOK(write, hooks_libc_hook)
#define readlink JEMALLOC_HOOK(readlink, hooks_libc_hook)
#define close JEMALLOC_HOOK(close, hooks_libc_hook)
#define creat JEMALLOC_HOOK(creat, hooks_libc_hook)
#define secure_getenv JEMALLOC_HOOK(secure_getenv, hooks_libc_hook)
/* Note that this is undef'd and re-define'd in src/prof.c. */
#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook)
#endif /* JEMALLOC_INTERNAL_HOOKS_H */
...@@ -31,6 +31,9 @@ ...@@ -31,6 +31,9 @@
# include <sys/uio.h> # include <sys/uio.h>
# endif # endif
# include <pthread.h> # include <pthread.h>
# ifdef __FreeBSD__
# include <pthread_np.h>
# endif
# include <signal.h> # include <signal.h>
# ifdef JEMALLOC_OS_UNFAIR_LOCK # ifdef JEMALLOC_OS_UNFAIR_LOCK
# include <os/lock.h> # include <os/lock.h>
......
...@@ -48,25 +48,13 @@ ...@@ -48,25 +48,13 @@
/* Defined if GCC __atomic atomics are available. */ /* Defined if GCC __atomic atomics are available. */
#undef JEMALLOC_GCC_ATOMIC_ATOMICS #undef JEMALLOC_GCC_ATOMIC_ATOMICS
/* and the 8-bit variant support. */
#undef JEMALLOC_GCC_U8_ATOMIC_ATOMICS
/* Defined if GCC __sync atomics are available. */ /* Defined if GCC __sync atomics are available. */
#undef JEMALLOC_GCC_SYNC_ATOMICS #undef JEMALLOC_GCC_SYNC_ATOMICS
/* and the 8-bit variant support. */
/* #undef JEMALLOC_GCC_U8_SYNC_ATOMICS
* Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and
* __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite
* __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the
* functions are defined in libgcc instead of being inlines).
*/
#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4
/*
* Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and
* __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite
* __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the
* functions are defined in libgcc instead of being inlines).
*/
#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8
/* /*
* Defined if __builtin_clz() and __builtin_clzl() are available. * Defined if __builtin_clz() and __builtin_clzl() are available.
...@@ -78,12 +66,6 @@ ...@@ -78,12 +66,6 @@
*/ */
#undef JEMALLOC_OS_UNFAIR_LOCK #undef JEMALLOC_OS_UNFAIR_LOCK
/*
* Defined if OSSpin*() functions are available, as provided by Darwin, and
* documented in the spinlock(3) manual page.
*/
#undef JEMALLOC_OSSPIN
/* Defined if syscall(2) is usable. */ /* Defined if syscall(2) is usable. */
#undef JEMALLOC_USE_SYSCALL #undef JEMALLOC_USE_SYSCALL
...@@ -153,6 +135,9 @@ ...@@ -153,6 +135,9 @@
/* JEMALLOC_STATS enables statistics calculation. */ /* JEMALLOC_STATS enables statistics calculation. */
#undef JEMALLOC_STATS #undef JEMALLOC_STATS
/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */
#undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API
/* JEMALLOC_PROF enables allocation profiling. */ /* JEMALLOC_PROF enables allocation profiling. */
#undef JEMALLOC_PROF #undef JEMALLOC_PROF
...@@ -233,6 +218,12 @@ ...@@ -233,6 +218,12 @@
#undef JEMALLOC_INTERNAL_FFSL #undef JEMALLOC_INTERNAL_FFSL
#undef JEMALLOC_INTERNAL_FFS #undef JEMALLOC_INTERNAL_FFS
/*
* popcount*() functions to use for bitmapping.
*/
#undef JEMALLOC_INTERNAL_POPCOUNTL
#undef JEMALLOC_INTERNAL_POPCOUNT
/* /*
* If defined, explicitly attempt to more uniformly distribute large allocation * If defined, explicitly attempt to more uniformly distribute large allocation
* pointer alignments across all cache indices. * pointer alignments across all cache indices.
...@@ -245,6 +236,12 @@ ...@@ -245,6 +236,12 @@
*/ */
#undef JEMALLOC_LOG #undef JEMALLOC_LOG
/*
* If defined, use readlinkat() (instead of readlink()) to follow
* /etc/malloc_conf.
*/
#undef JEMALLOC_READLINKAT
/* /*
* Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
*/ */
...@@ -363,4 +360,7 @@ ...@@ -363,4 +360,7 @@
*/ */
#undef JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE #undef JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE
/* Performs additional safety checks when defined. */
#undef JEMALLOC_OPT_SAFETY_CHECKS
#endif /* JEMALLOC_INTERNAL_DEFS_H_ */ #endif /* JEMALLOC_INTERNAL_DEFS_H_ */
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
#define JEMALLOC_INTERNAL_EXTERNS_H #define JEMALLOC_INTERNAL_EXTERNS_H
#include "jemalloc/internal/atomic.h" #include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/tsd_types.h" #include "jemalloc/internal/tsd_types.h"
/* TSD checks this to set thread local slow state accordingly. */ /* TSD checks this to set thread local slow state accordingly. */
...@@ -11,6 +10,7 @@ extern bool malloc_slow; ...@@ -11,6 +10,7 @@ extern bool malloc_slow;
/* Run-time options. */ /* Run-time options. */
extern bool opt_abort; extern bool opt_abort;
extern bool opt_abort_conf; extern bool opt_abort_conf;
extern bool opt_confirm_conf;
extern const char *opt_junk; extern const char *opt_junk;
extern bool opt_junk_alloc; extern bool opt_junk_alloc;
extern bool opt_junk_free; extern bool opt_junk_free;
...@@ -25,6 +25,9 @@ extern unsigned ncpus; ...@@ -25,6 +25,9 @@ extern unsigned ncpus;
/* Number of arenas used for automatic multiplexing of threads and arenas. */ /* Number of arenas used for automatic multiplexing of threads and arenas. */
extern unsigned narenas_auto; extern unsigned narenas_auto;
/* Base index for manual arenas. */
extern unsigned manual_arena_base;
/* /*
* Arenas that are used to service external requests. Not all elements of the * Arenas that are used to service external requests. Not all elements of the
* arenas array are necessarily used; arenas are created lazily as needed. * arenas array are necessarily used; arenas are created lazily as needed.
...@@ -49,5 +52,6 @@ void jemalloc_prefork(void); ...@@ -49,5 +52,6 @@ void jemalloc_prefork(void);
void jemalloc_postfork_parent(void); void jemalloc_postfork_parent(void);
void jemalloc_postfork_child(void); void jemalloc_postfork_child(void);
bool malloc_initialized(void); bool malloc_initialized(void);
void je_sdallocx_noflags(void *ptr, size_t size);
#endif /* JEMALLOC_INTERNAL_EXTERNS_H */ #endif /* JEMALLOC_INTERNAL_EXTERNS_H */
...@@ -4,13 +4,15 @@ ...@@ -4,13 +4,15 @@
#include "jemalloc/internal/atomic.h" #include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/sc.h"
#include "jemalloc/internal/ticker.h" #include "jemalloc/internal/ticker.h"
JEMALLOC_ALWAYS_INLINE malloc_cpuid_t JEMALLOC_ALWAYS_INLINE malloc_cpuid_t
malloc_getcpu(void) { malloc_getcpu(void) {
assert(have_percpu_arena); assert(have_percpu_arena);
#if defined(JEMALLOC_HAVE_SCHED_GETCPU) #if defined(_WIN32)
return GetCurrentProcessorNumber();
#elif defined(JEMALLOC_HAVE_SCHED_GETCPU)
return (malloc_cpuid_t)sched_getcpu(); return (malloc_cpuid_t)sched_getcpu();
#else #else
not_reached(); not_reached();
...@@ -108,14 +110,14 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) { ...@@ -108,14 +110,14 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) {
JEMALLOC_ALWAYS_INLINE cache_bin_t * JEMALLOC_ALWAYS_INLINE cache_bin_t *
tcache_small_bin_get(tcache_t *tcache, szind_t binind) { tcache_small_bin_get(tcache_t *tcache, szind_t binind) {
assert(binind < NBINS); assert(binind < SC_NBINS);
return &tcache->bins_small[binind]; return &tcache->bins_small[binind];
} }
JEMALLOC_ALWAYS_INLINE cache_bin_t * JEMALLOC_ALWAYS_INLINE cache_bin_t *
tcache_large_bin_get(tcache_t *tcache, szind_t binind) { tcache_large_bin_get(tcache_t *tcache, szind_t binind) {
assert(binind >= NBINS &&binind < nhbins); assert(binind >= SC_NBINS &&binind < nhbins);
return &tcache->bins_large[binind - NBINS]; return &tcache->bins_large[binind - SC_NBINS];
} }
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
...@@ -156,7 +158,7 @@ pre_reentrancy(tsd_t *tsd, arena_t *arena) { ...@@ -156,7 +158,7 @@ pre_reentrancy(tsd_t *tsd, arena_t *arena) {
if (fast) { if (fast) {
/* Prepare slow path for reentrancy. */ /* Prepare slow path for reentrancy. */
tsd_slow_update(tsd); tsd_slow_update(tsd);
assert(tsd->state == tsd_state_nominal_slow); assert(tsd_state_get(tsd) == tsd_state_nominal_slow);
} }
} }
......
...@@ -71,7 +71,8 @@ arena_ichoose(tsd_t *tsd, arena_t *arena) { ...@@ -71,7 +71,8 @@ arena_ichoose(tsd_t *tsd, arena_t *arena) {
static inline bool static inline bool
arena_is_auto(arena_t *arena) { arena_is_auto(arena_t *arena) {
assert(narenas_auto > 0); assert(narenas_auto > 0);
return (arena_ind_get(arena) < narenas_auto);
return (arena_ind_get(arena) < manual_arena_base);
} }
JEMALLOC_ALWAYS_INLINE extent_t * JEMALLOC_ALWAYS_INLINE extent_t *
......
...@@ -54,7 +54,7 @@ size_t malloc_vsnprintf(char *str, size_t size, const char *format, ...@@ -54,7 +54,7 @@ size_t malloc_vsnprintf(char *str, size_t size, const char *format,
size_t malloc_snprintf(char *str, size_t size, const char *format, ...) size_t malloc_snprintf(char *str, size_t size, const char *format, ...)
JEMALLOC_FORMAT_PRINTF(3, 4); JEMALLOC_FORMAT_PRINTF(3, 4);
/* /*
* The caller can set write_cb and cbopaque to null to choose to print with the * The caller can set write_cb to null to choose to print with the
* je_malloc_message hook. * je_malloc_message hook.
*/ */
void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
......
...@@ -169,7 +169,6 @@ struct prof_tdata_s { ...@@ -169,7 +169,6 @@ struct prof_tdata_s {
/* Sampling state. */ /* Sampling state. */
uint64_t prng_state; uint64_t prng_state;
uint64_t bytes_until_sample;
/* State used to avoid dumping while operating on prof internals. */ /* State used to avoid dumping while operating on prof internals. */
bool enq; bool enq;
......
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment