Unverified Commit c4b4b6c0 authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Merge pull request #9623 from yoav-steinberg/upgrade_jemalloc_5.2.1

Upgraded to jemalloc 5.2.1 from 5.1.0.
Cherry picked all relevant fixes (by diffing our 5.1.0 to upstream 5.10 and finding relevant commits).
Details of what was done:

[cherry-picked] fd7d51c3 2021-05-03 Resolve nonsense static analysis warnings (Oran Agra)
[cherry-picked] 448c435b 2020-09-29 Fix compilation warnings in Lua and jemalloc dependencies (#7785) (YoongHM)
[skipped - already in upstream] 9216b96b 2020-09-21 Fix compilation warning in jemalloc's malloc_vsnprintf (#7789) (YoongHM)
[cherry-picked] 88d71f47 2020-05-20 fix a rare active defrag edge case bug leading to stagnation (Oran Agra)
[skipped - already in upstream] 2fec7d9c 2019-05-30 Jemalloc: Avoid blocking on background thread lock for stats.
[cherry-picked] 920158ec 2018-07-11 Active defrag fixes for 32bit builds (again) (Oran Agra)
[cherry-picked] e8099cab 2018-06-26 add defrag hint support into jemalloc 5 (Oran Agra)
[re-done] 4e729fcd 2018-05-24 Generate configure for Jemalloc. (antirez)

Additionally had to do this:
7727cc2 2021-10-10 Fix defrag to support sharded bins in arena (added in v5.2.1) (Yoav Steinberg)

When reviewing please look at all except the first commit which is just replacing 5.1.0 with 5.2.1 sources.
Also I think we should merge this without squashing to preserve the changes we did to to jemalloc.
parents 276b460e 85737e67
#ifndef JEMALLOC_INTERNAL_ATOMIC_H #ifndef JEMALLOC_INTERNAL_ATOMIC_H
#define JEMALLOC_INTERNAL_ATOMIC_H #define JEMALLOC_INTERNAL_ATOMIC_H
#define ATOMIC_INLINE static inline #define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE
#define JEMALLOC_U8_ATOMICS
#if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) #if defined(JEMALLOC_GCC_ATOMIC_ATOMICS)
# include "jemalloc/internal/atomic_gcc_atomic.h" # include "jemalloc/internal/atomic_gcc_atomic.h"
# if !defined(JEMALLOC_GCC_U8_ATOMIC_ATOMICS)
# undef JEMALLOC_U8_ATOMICS
# endif
#elif defined(JEMALLOC_GCC_SYNC_ATOMICS) #elif defined(JEMALLOC_GCC_SYNC_ATOMICS)
# include "jemalloc/internal/atomic_gcc_sync.h" # include "jemalloc/internal/atomic_gcc_sync.h"
# if !defined(JEMALLOC_GCC_U8_SYNC_ATOMICS)
# undef JEMALLOC_U8_ATOMICS
# endif
#elif defined(_MSC_VER) #elif defined(_MSC_VER)
# include "jemalloc/internal/atomic_msvc.h" # include "jemalloc/internal/atomic_msvc.h"
#elif defined(JEMALLOC_C11_ATOMICS) #elif defined(JEMALLOC_C11_ATOMICS)
...@@ -66,6 +73,8 @@ JEMALLOC_GENERATE_INT_ATOMICS(size_t, zu, LG_SIZEOF_PTR) ...@@ -66,6 +73,8 @@ JEMALLOC_GENERATE_INT_ATOMICS(size_t, zu, LG_SIZEOF_PTR)
JEMALLOC_GENERATE_INT_ATOMICS(ssize_t, zd, LG_SIZEOF_PTR) JEMALLOC_GENERATE_INT_ATOMICS(ssize_t, zd, LG_SIZEOF_PTR)
JEMALLOC_GENERATE_INT_ATOMICS(uint8_t, u8, 0)
JEMALLOC_GENERATE_INT_ATOMICS(uint32_t, u32, 2) JEMALLOC_GENERATE_INT_ATOMICS(uint32_t, u32, 2)
#ifdef JEMALLOC_ATOMIC_U64 #ifdef JEMALLOC_ATOMIC_U64
......
...@@ -67,7 +67,8 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ ...@@ -67,7 +67,8 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
\ \
ATOMIC_INLINE bool \ ATOMIC_INLINE bool \
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \ UNUSED type *expected, type desired, \
atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \ atomic_memory_order_t failure_mo) { \
return __atomic_compare_exchange(&a->repr, expected, &desired, \ return __atomic_compare_exchange(&a->repr, expected, &desired, \
true, atomic_enum_to_builtin(success_mo), \ true, atomic_enum_to_builtin(success_mo), \
...@@ -76,7 +77,8 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ ...@@ -76,7 +77,8 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
\ \
ATOMIC_INLINE bool \ ATOMIC_INLINE bool \
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \ UNUSED type *expected, type desired, \
atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \ atomic_memory_order_t failure_mo) { \
return __atomic_compare_exchange(&a->repr, expected, &desired, \ return __atomic_compare_exchange(&a->repr, expected, &desired, \
false, \ false, \
......
...@@ -27,8 +27,10 @@ atomic_fence(atomic_memory_order_t mo) { ...@@ -27,8 +27,10 @@ atomic_fence(atomic_memory_order_t mo) {
asm volatile("" ::: "memory"); asm volatile("" ::: "memory");
# if defined(__i386__) || defined(__x86_64__) # if defined(__i386__) || defined(__x86_64__)
/* This is implicit on x86. */ /* This is implicit on x86. */
# elif defined(__ppc__) # elif defined(__ppc64__)
asm volatile("lwsync"); asm volatile("lwsync");
# elif defined(__ppc__)
asm volatile("sync");
# elif defined(__sparc__) && defined(__arch64__) # elif defined(__sparc__) && defined(__arch64__)
if (mo == atomic_memory_order_acquire) { if (mo == atomic_memory_order_acquire) {
asm volatile("membar #LoadLoad | #LoadStore"); asm volatile("membar #LoadLoad | #LoadStore");
...@@ -113,8 +115,8 @@ atomic_store_##short_type(atomic_##short_type##_t *a, \ ...@@ -113,8 +115,8 @@ atomic_store_##short_type(atomic_##short_type##_t *a, \
} \ } \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type \
atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \ atomic_memory_order_t mo) { \
/* \ /* \
* Because of FreeBSD, we care about gcc 4.2, which doesn't have\ * Because of FreeBSD, we care about gcc 4.2, which doesn't have\
* an atomic exchange builtin. We fake it with a CAS loop. \ * an atomic exchange builtin. We fake it with a CAS loop. \
...@@ -129,8 +131,9 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ ...@@ -129,8 +131,9 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
\ \
ATOMIC_INLINE bool \ ATOMIC_INLINE bool \
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \ type *expected, type desired, \
atomic_memory_order_t failure_mo) { \ atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
type prev = __sync_val_compare_and_swap(&a->repr, *expected, \ type prev = __sync_val_compare_and_swap(&a->repr, *expected, \
desired); \ desired); \
if (prev == *expected) { \ if (prev == *expected) { \
...@@ -142,8 +145,9 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ ...@@ -142,8 +145,9 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
} \ } \
ATOMIC_INLINE bool \ ATOMIC_INLINE bool \
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \ type *expected, type desired, \
atomic_memory_order_t failure_mo) { \ atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
type prev = __sync_val_compare_and_swap(&a->repr, *expected, \ type prev = __sync_val_compare_and_swap(&a->repr, *expected, \
desired); \ desired); \
if (prev == *expected) { \ if (prev == *expected) { \
......
...@@ -8,7 +8,6 @@ extern atomic_b_t background_thread_enabled_state; ...@@ -8,7 +8,6 @@ extern atomic_b_t background_thread_enabled_state;
extern size_t n_background_threads; extern size_t n_background_threads;
extern size_t max_background_threads; extern size_t max_background_threads;
extern background_thread_info_t *background_thread_info; extern background_thread_info_t *background_thread_info;
extern bool can_enable_background_thread;
bool background_thread_create(tsd_t *tsd, unsigned arena_ind); bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
bool background_threads_enable(tsd_t *tsd); bool background_threads_enable(tsd_t *tsd);
......
...@@ -15,7 +15,12 @@ background_thread_enabled_set(tsdn_t *tsdn, bool state) { ...@@ -15,7 +15,12 @@ background_thread_enabled_set(tsdn_t *tsdn, bool state) {
JEMALLOC_ALWAYS_INLINE background_thread_info_t * JEMALLOC_ALWAYS_INLINE background_thread_info_t *
arena_background_thread_info_get(arena_t *arena) { arena_background_thread_info_get(arena_t *arena) {
unsigned arena_ind = arena_ind_get(arena); unsigned arena_ind = arena_ind_get(arena);
return &background_thread_info[arena_ind % ncpus]; return &background_thread_info[arena_ind % max_background_threads];
}
JEMALLOC_ALWAYS_INLINE background_thread_info_t *
background_thread_info_get(size_t ind) {
return &background_thread_info[ind % max_background_threads];
} }
JEMALLOC_ALWAYS_INLINE uint64_t JEMALLOC_ALWAYS_INLINE uint64_t
......
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
#define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX #define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX
#define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT #define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT
#define DEFAULT_NUM_BACKGROUND_THREAD 4
typedef enum { typedef enum {
background_thread_stopped, background_thread_stopped,
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
#include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/sc.h"
/* Embedded at the beginning of every block of base-managed virtual memory. */ /* Embedded at the beginning of every block of base-managed virtual memory. */
struct base_block_s { struct base_block_s {
...@@ -46,7 +46,7 @@ struct base_s { ...@@ -46,7 +46,7 @@ struct base_s {
base_block_t *blocks; base_block_t *blocks;
/* Heap of extents that track unused trailing space within blocks. */ /* Heap of extents that track unused trailing space within blocks. */
extent_heap_t avail[NSIZES]; extent_heap_t avail[SC_NSIZES];
/* Stats, only maintained if config_stats. */ /* Stats, only maintained if config_stats. */
size_t allocated; size_t allocated;
......
#ifndef JEMALLOC_INTERNAL_BIN_H #ifndef JEMALLOC_INTERNAL_BIN_H
#define JEMALLOC_INTERNAL_BIN_H #define JEMALLOC_INTERNAL_BIN_H
#include "jemalloc/internal/bin_stats.h"
#include "jemalloc/internal/bin_types.h"
#include "jemalloc/internal/extent_types.h" #include "jemalloc/internal/extent_types.h"
#include "jemalloc/internal/extent_structs.h" #include "jemalloc/internal/extent_structs.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/bin_stats.h" #include "jemalloc/internal/sc.h"
/* /*
* A bin contains a set of extents that are currently being used for slab * A bin contains a set of extents that are currently being used for slab
...@@ -41,6 +43,9 @@ struct bin_info_s { ...@@ -41,6 +43,9 @@ struct bin_info_s {
/* Total number of regions in a slab for this bin's size class. */ /* Total number of regions in a slab for this bin's size class. */
uint32_t nregs; uint32_t nregs;
/* Number of sharded bins in each arena for this size class. */
uint32_t n_shards;
/* /*
* Metadata used to manipulate bitmaps for slabs associated with this * Metadata used to manipulate bitmaps for slabs associated with this
* bin. * bin.
...@@ -48,8 +53,7 @@ struct bin_info_s { ...@@ -48,8 +53,7 @@ struct bin_info_s {
bitmap_info_t bitmap_info; bitmap_info_t bitmap_info;
}; };
extern const bin_info_t bin_infos[NBINS]; extern bin_info_t bin_infos[SC_NBINS];
typedef struct bin_s bin_t; typedef struct bin_s bin_t;
struct bin_s { struct bin_s {
...@@ -78,6 +82,18 @@ struct bin_s { ...@@ -78,6 +82,18 @@ struct bin_s {
bin_stats_t stats; bin_stats_t stats;
}; };
/* A set of sharded bins of the same size class. */
typedef struct bins_s bins_t;
struct bins_s {
/* Sharded bins. Dynamically sized. */
bin_t *bin_shards;
};
void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]);
bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size,
size_t end_size, size_t nshards);
void bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]);
/* Initializes a bin to empty. Returns true on error. */ /* Initializes a bin to empty. Returns true on error. */
bool bin_init(bin_t *bin); bool bin_init(bin_t *bin);
...@@ -90,7 +106,7 @@ void bin_postfork_child(tsdn_t *tsdn, bin_t *bin); ...@@ -90,7 +106,7 @@ void bin_postfork_child(tsdn_t *tsdn, bin_t *bin);
static inline void static inline void
bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) { bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) {
malloc_mutex_lock(tsdn, &bin->lock); malloc_mutex_lock(tsdn, &bin->lock);
malloc_mutex_prof_read(tsdn, &dst_bin_stats->mutex_data, &bin->lock); malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock);
dst_bin_stats->nmalloc += bin->stats.nmalloc; dst_bin_stats->nmalloc += bin->stats.nmalloc;
dst_bin_stats->ndalloc += bin->stats.ndalloc; dst_bin_stats->ndalloc += bin->stats.ndalloc;
dst_bin_stats->nrequests += bin->stats.nrequests; dst_bin_stats->nrequests += bin->stats.nrequests;
...@@ -100,6 +116,7 @@ bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) { ...@@ -100,6 +116,7 @@ bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) {
dst_bin_stats->nslabs += bin->stats.nslabs; dst_bin_stats->nslabs += bin->stats.nslabs;
dst_bin_stats->reslabs += bin->stats.reslabs; dst_bin_stats->reslabs += bin->stats.reslabs;
dst_bin_stats->curslabs += bin->stats.curslabs; dst_bin_stats->curslabs += bin->stats.curslabs;
dst_bin_stats->nonfull_slabs += bin->stats.nonfull_slabs;
malloc_mutex_unlock(tsdn, &bin->lock); malloc_mutex_unlock(tsdn, &bin->lock);
} }
......
...@@ -45,6 +45,9 @@ struct bin_stats_s { ...@@ -45,6 +45,9 @@ struct bin_stats_s {
/* Current number of slabs in this bin. */ /* Current number of slabs in this bin. */
size_t curslabs; size_t curslabs;
/* Current size of nonfull slabs heap in this bin. */
size_t nonfull_slabs;
mutex_prof_data_t mutex_data; mutex_prof_data_t mutex_data;
}; };
......
#ifndef JEMALLOC_INTERNAL_BIN_TYPES_H
#define JEMALLOC_INTERNAL_BIN_TYPES_H
#include "jemalloc/internal/sc.h"
#define BIN_SHARDS_MAX (1 << EXTENT_BITS_BINSHARD_WIDTH)
#define N_BIN_SHARDS_DEFAULT 1
/* Used in TSD static initializer only. Real init in arena_bind(). */
#define TSD_BINSHARDS_ZERO_INITIALIZER {{UINT8_MAX}}
typedef struct tsd_binshards_s tsd_binshards_t;
struct tsd_binshards_s {
uint8_t binshard[SC_NBINS];
};
#endif /* JEMALLOC_INTERNAL_BIN_TYPES_H */
...@@ -27,6 +27,25 @@ ffs_u(unsigned bitmap) { ...@@ -27,6 +27,25 @@ ffs_u(unsigned bitmap) {
return JEMALLOC_INTERNAL_FFS(bitmap); return JEMALLOC_INTERNAL_FFS(bitmap);
} }
#ifdef JEMALLOC_INTERNAL_POPCOUNTL
BIT_UTIL_INLINE unsigned
popcount_lu(unsigned long bitmap) {
return JEMALLOC_INTERNAL_POPCOUNTL(bitmap);
}
#endif
/*
* Clears first unset bit in bitmap, and returns
* place of bit. bitmap *must not* be 0.
*/
BIT_UTIL_INLINE size_t
cfs_lu(unsigned long* bitmap) {
size_t bit = ffs_lu(*bitmap) - 1;
*bitmap ^= ZU(1) << bit;
return bit;
}
BIT_UTIL_INLINE unsigned BIT_UTIL_INLINE unsigned
ffs_zu(size_t bitmap) { ffs_zu(size_t bitmap) {
#if LG_SIZEOF_PTR == LG_SIZEOF_INT #if LG_SIZEOF_PTR == LG_SIZEOF_INT
...@@ -63,6 +82,22 @@ ffs_u32(uint32_t bitmap) { ...@@ -63,6 +82,22 @@ ffs_u32(uint32_t bitmap) {
BIT_UTIL_INLINE uint64_t BIT_UTIL_INLINE uint64_t
pow2_ceil_u64(uint64_t x) { pow2_ceil_u64(uint64_t x) {
#if (defined(__amd64__) || defined(__x86_64__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ))
if(unlikely(x <= 1)) {
return x;
}
size_t msb_on_index;
#if (defined(__amd64__) || defined(__x86_64__))
asm ("bsrq %1, %0"
: "=r"(msb_on_index) // Outputs.
: "r"(x-1) // Inputs.
);
#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
msb_on_index = (63 ^ __builtin_clzll(x - 1));
#endif
assert(msb_on_index < 63);
return 1ULL << (msb_on_index + 1);
#else
x--; x--;
x |= x >> 1; x |= x >> 1;
x |= x >> 2; x |= x >> 2;
...@@ -72,10 +107,27 @@ pow2_ceil_u64(uint64_t x) { ...@@ -72,10 +107,27 @@ pow2_ceil_u64(uint64_t x) {
x |= x >> 32; x |= x >> 32;
x++; x++;
return x; return x;
#endif
} }
BIT_UTIL_INLINE uint32_t BIT_UTIL_INLINE uint32_t
pow2_ceil_u32(uint32_t x) { pow2_ceil_u32(uint32_t x) {
#if ((defined(__i386__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ)) && (!defined(__s390__)))
if(unlikely(x <= 1)) {
return x;
}
size_t msb_on_index;
#if (defined(__i386__))
asm ("bsr %1, %0"
: "=r"(msb_on_index) // Outputs.
: "r"(x-1) // Inputs.
);
#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
msb_on_index = (31 ^ __builtin_clz(x - 1));
#endif
assert(msb_on_index < 31);
return 1U << (msb_on_index + 1);
#else
x--; x--;
x |= x >> 1; x |= x >> 1;
x |= x >> 2; x |= x >> 2;
...@@ -84,6 +136,7 @@ pow2_ceil_u32(uint32_t x) { ...@@ -84,6 +136,7 @@ pow2_ceil_u32(uint32_t x) {
x |= x >> 16; x |= x >> 16;
x++; x++;
return x; return x;
#endif
} }
/* Compute the smallest power of 2 that is >= x. */ /* Compute the smallest power of 2 that is >= x. */
...@@ -160,6 +213,27 @@ lg_floor(size_t x) { ...@@ -160,6 +213,27 @@ lg_floor(size_t x) {
} }
#endif #endif
BIT_UTIL_INLINE unsigned
lg_ceil(size_t x) {
return lg_floor(x) + ((x & (x - 1)) == 0 ? 0 : 1);
}
#undef BIT_UTIL_INLINE #undef BIT_UTIL_INLINE
/* A compile-time version of lg_floor and lg_ceil. */
#define LG_FLOOR_1(x) 0
#define LG_FLOOR_2(x) (x < (1ULL << 1) ? LG_FLOOR_1(x) : 1 + LG_FLOOR_1(x >> 1))
#define LG_FLOOR_4(x) (x < (1ULL << 2) ? LG_FLOOR_2(x) : 2 + LG_FLOOR_2(x >> 2))
#define LG_FLOOR_8(x) (x < (1ULL << 4) ? LG_FLOOR_4(x) : 4 + LG_FLOOR_4(x >> 4))
#define LG_FLOOR_16(x) (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8))
#define LG_FLOOR_32(x) (x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16))
#define LG_FLOOR_64(x) (x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32))
#if LG_SIZEOF_PTR == 2
# define LG_FLOOR(x) LG_FLOOR_32((x))
#else
# define LG_FLOOR(x) LG_FLOOR_64((x))
#endif
#define LG_CEIL(x) (LG_FLOOR(x) + (((x) & ((x) - 1)) == 0 ? 0 : 1))
#endif /* JEMALLOC_INTERNAL_BIT_UTIL_H */ #endif /* JEMALLOC_INTERNAL_BIT_UTIL_H */
...@@ -3,18 +3,18 @@ ...@@ -3,18 +3,18 @@
#include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/arena_types.h"
#include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/sc.h"
typedef unsigned long bitmap_t; typedef unsigned long bitmap_t;
#define LG_SIZEOF_BITMAP LG_SIZEOF_LONG #define LG_SIZEOF_BITMAP LG_SIZEOF_LONG
/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
#if LG_SLAB_MAXREGS > LG_CEIL_NSIZES #if LG_SLAB_MAXREGS > LG_CEIL(SC_NSIZES)
/* Maximum bitmap bit count is determined by maximum regions per slab. */ /* Maximum bitmap bit count is determined by maximum regions per slab. */
# define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS # define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS
#else #else
/* Maximum bitmap bit count is determined by number of extent size classes. */ /* Maximum bitmap bit count is determined by number of extent size classes. */
# define LG_BITMAP_MAXBITS LG_CEIL_NSIZES # define LG_BITMAP_MAXBITS LG_CEIL(SC_NSIZES)
#endif #endif
#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS) #define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS)
......
...@@ -88,11 +88,21 @@ JEMALLOC_ALWAYS_INLINE void * ...@@ -88,11 +88,21 @@ JEMALLOC_ALWAYS_INLINE void *
cache_bin_alloc_easy(cache_bin_t *bin, bool *success) { cache_bin_alloc_easy(cache_bin_t *bin, bool *success) {
void *ret; void *ret;
if (unlikely(bin->ncached == 0)) { bin->ncached--;
bin->low_water = -1;
*success = false; /*
return NULL; * Check for both bin->ncached == 0 and ncached < low_water
* in a single branch.
*/
if (unlikely(bin->ncached <= bin->low_water)) {
bin->low_water = bin->ncached;
if (bin->ncached == -1) {
bin->ncached = 0;
*success = false;
return NULL;
}
} }
/* /*
* success (instead of ret) should be checked upon the return of this * success (instead of ret) should be checked upon the return of this
* function. We avoid checking (ret == NULL) because there is never a * function. We avoid checking (ret == NULL) because there is never a
...@@ -101,14 +111,21 @@ cache_bin_alloc_easy(cache_bin_t *bin, bool *success) { ...@@ -101,14 +111,21 @@ cache_bin_alloc_easy(cache_bin_t *bin, bool *success) {
* cacheline). * cacheline).
*/ */
*success = true; *success = true;
ret = *(bin->avail - bin->ncached); ret = *(bin->avail - (bin->ncached + 1));
bin->ncached--;
if (unlikely(bin->ncached < bin->low_water)) { return ret;
bin->low_water = bin->ncached; }
JEMALLOC_ALWAYS_INLINE bool
cache_bin_dalloc_easy(cache_bin_t *bin, cache_bin_info_t *bin_info, void *ptr) {
if (unlikely(bin->ncached == bin_info->ncached_max)) {
return false;
} }
assert(bin->ncached < bin_info->ncached_max);
bin->ncached++;
*(bin->avail - bin->ncached) = ptr;
return ret; return true;
} }
#endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */ #endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
#include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex_prof.h" #include "jemalloc/internal/mutex_prof.h"
#include "jemalloc/internal/ql.h" #include "jemalloc/internal/ql.h"
#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/sc.h"
#include "jemalloc/internal/stats.h" #include "jemalloc/internal/stats.h"
/* Maximum ctl tree depth. */ /* Maximum ctl tree depth. */
...@@ -39,9 +39,12 @@ typedef struct ctl_arena_stats_s { ...@@ -39,9 +39,12 @@ typedef struct ctl_arena_stats_s {
uint64_t nmalloc_small; uint64_t nmalloc_small;
uint64_t ndalloc_small; uint64_t ndalloc_small;
uint64_t nrequests_small; uint64_t nrequests_small;
uint64_t nfills_small;
uint64_t nflushes_small;
bin_stats_t bstats[NBINS]; bin_stats_t bstats[SC_NBINS];
arena_stats_large_t lstats[NSIZES - NBINS]; arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
arena_stats_extents_t estats[SC_NPSIZES];
} ctl_arena_stats_t; } ctl_arena_stats_t;
typedef struct ctl_stats_s { typedef struct ctl_stats_s {
......
...@@ -45,7 +45,9 @@ struct emitter_col_s { ...@@ -45,7 +45,9 @@ struct emitter_col_s {
int int_val; int int_val;
unsigned unsigned_val; unsigned unsigned_val;
uint32_t uint32_val; uint32_t uint32_val;
uint32_t uint32_t_val;
uint64_t uint64_val; uint64_t uint64_val;
uint64_t uint64_t_val;
size_t size_val; size_t size_val;
ssize_t ssize_val; ssize_t ssize_val;
const char *str_val; const char *str_val;
...@@ -60,17 +62,6 @@ struct emitter_row_s { ...@@ -60,17 +62,6 @@ struct emitter_row_s {
ql_head(emitter_col_t) cols; ql_head(emitter_col_t) cols;
}; };
static inline void
emitter_row_init(emitter_row_t *row) {
ql_new(&row->cols);
}
static inline void
emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
ql_elm_new(col, link);
ql_tail_insert(&row->cols, col, link);
}
typedef struct emitter_s emitter_t; typedef struct emitter_s emitter_t;
struct emitter_s { struct emitter_s {
emitter_output_t output; emitter_output_t output;
...@@ -80,18 +71,10 @@ struct emitter_s { ...@@ -80,18 +71,10 @@ struct emitter_s {
int nesting_depth; int nesting_depth;
/* True if we've already emitted a value at the given depth. */ /* True if we've already emitted a value at the given depth. */
bool item_at_depth; bool item_at_depth;
/* True if we emitted a key and will emit corresponding value next. */
bool emitted_key;
}; };
static inline void
emitter_init(emitter_t *emitter, emitter_output_t emitter_output,
void (*write_cb)(void *, const char *), void *cbopaque) {
emitter->output = emitter_output;
emitter->write_cb = write_cb;
emitter->cbopaque = cbopaque;
emitter->item_at_depth = false;
emitter->nesting_depth = 0;
}
/* Internal convenience function. Write to the emitter the given string. */ /* Internal convenience function. Write to the emitter the given string. */
JEMALLOC_FORMAT_PRINTF(2, 3) JEMALLOC_FORMAT_PRINTF(2, 3)
static inline void static inline void
...@@ -103,22 +86,11 @@ emitter_printf(emitter_t *emitter, const char *format, ...) { ...@@ -103,22 +86,11 @@ emitter_printf(emitter_t *emitter, const char *format, ...) {
va_end(ap); va_end(ap);
} }
/* Write to the emitter the given string, but only in table mode. */ static inline const char * JEMALLOC_FORMAT_ARG(3)
JEMALLOC_FORMAT_PRINTF(2, 3)
static inline void
emitter_table_printf(emitter_t *emitter, const char *format, ...) {
if (emitter->output == emitter_output_table) {
va_list ap;
va_start(ap, format);
malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap);
va_end(ap);
}
}
static inline void
emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier,
emitter_justify_t justify, int width) { emitter_justify_t justify, int width) {
size_t written; size_t written;
fmt_specifier++;
if (justify == emitter_justify_none) { if (justify == emitter_justify_none) {
written = malloc_snprintf(out_fmt, out_size, written = malloc_snprintf(out_fmt, out_size,
"%%%s", fmt_specifier); "%%%s", fmt_specifier);
...@@ -131,6 +103,7 @@ emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, ...@@ -131,6 +103,7 @@ emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier,
} }
/* Only happens in case of bad format string, which *we* choose. */ /* Only happens in case of bad format string, which *we* choose. */
assert(written < out_size); assert(written < out_size);
return out_fmt;
} }
/* /*
...@@ -156,26 +129,27 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, ...@@ -156,26 +129,27 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
char buf[BUF_SIZE]; char buf[BUF_SIZE];
#define EMIT_SIMPLE(type, format) \ #define EMIT_SIMPLE(type, format) \
emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width); \ emitter_printf(emitter, \
emitter_printf(emitter, fmt, *(const type *)value); \ emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width), \
*(const type *)value);
switch (value_type) { switch (value_type) {
case emitter_type_bool: case emitter_type_bool:
emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width); emitter_printf(emitter,
emitter_printf(emitter, fmt, *(const bool *)value ? emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width),
"true" : "false"); *(const bool *)value ? "true" : "false");
break; break;
case emitter_type_int: case emitter_type_int:
EMIT_SIMPLE(int, "d") EMIT_SIMPLE(int, "%d")
break; break;
case emitter_type_unsigned: case emitter_type_unsigned:
EMIT_SIMPLE(unsigned, "u") EMIT_SIMPLE(unsigned, "%u")
break; break;
case emitter_type_ssize: case emitter_type_ssize:
EMIT_SIMPLE(ssize_t, "zd") EMIT_SIMPLE(ssize_t, "%zd")
break; break;
case emitter_type_size: case emitter_type_size:
EMIT_SIMPLE(size_t, "zu") EMIT_SIMPLE(size_t, "%zu")
break; break;
case emitter_type_string: case emitter_type_string:
str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"", str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"",
...@@ -185,17 +159,17 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, ...@@ -185,17 +159,17 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
* anywhere near the fmt size. * anywhere near the fmt size.
*/ */
assert(str_written < BUF_SIZE); assert(str_written < BUF_SIZE);
emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width); emitter_printf(emitter,
emitter_printf(emitter, fmt, buf); emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), buf);
break; break;
case emitter_type_uint32: case emitter_type_uint32:
EMIT_SIMPLE(uint32_t, FMTu32) EMIT_SIMPLE(uint32_t, "%" FMTu32)
break; break;
case emitter_type_uint64: case emitter_type_uint64:
EMIT_SIMPLE(uint64_t, FMTu64) EMIT_SIMPLE(uint64_t, "%" FMTu64)
break; break;
case emitter_type_title: case emitter_type_title:
EMIT_SIMPLE(char *const, "s"); EMIT_SIMPLE(char *const, "%s");
break; break;
default: default:
unreachable(); unreachable();
...@@ -235,201 +209,278 @@ emitter_indent(emitter_t *emitter) { ...@@ -235,201 +209,278 @@ emitter_indent(emitter_t *emitter) {
static inline void static inline void
emitter_json_key_prefix(emitter_t *emitter) { emitter_json_key_prefix(emitter_t *emitter) {
if (emitter->emitted_key) {
emitter->emitted_key = false;
return;
}
emitter_printf(emitter, "%s\n", emitter->item_at_depth ? "," : ""); emitter_printf(emitter, "%s\n", emitter->item_at_depth ? "," : "");
emitter_indent(emitter); emitter_indent(emitter);
} }
/******************************************************************************/
/* Public functions for emitter_t. */
static inline void static inline void
emitter_begin(emitter_t *emitter) { emitter_init(emitter_t *emitter, emitter_output_t emitter_output,
if (emitter->output == emitter_output_json) { void (*write_cb)(void *, const char *), void *cbopaque) {
assert(emitter->nesting_depth == 0); emitter->output = emitter_output;
emitter_printf(emitter, "{"); emitter->write_cb = write_cb;
emitter_nest_inc(emitter); emitter->cbopaque = cbopaque;
} else { emitter->item_at_depth = false;
// tabular init emitter->emitted_key = false;
emitter_printf(emitter, "%s", ""); emitter->nesting_depth = 0;
}
} }
/******************************************************************************/
/* JSON public API. */
/*
* Emits a key (e.g. as appears in an object). The next json entity emitted will
* be the corresponding value.
*/
static inline void static inline void
emitter_end(emitter_t *emitter) { emitter_json_key(emitter_t *emitter, const char *json_key) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth == 1); emitter_json_key_prefix(emitter);
emitter_nest_dec(emitter); emitter_printf(emitter, "\"%s\": ", json_key);
emitter_printf(emitter, "\n}\n"); emitter->emitted_key = true;
} }
} }
/*
* Note emits a different kv pair as well, but only in table mode. Omits the
* note if table_note_key is NULL.
*/
static inline void static inline void
emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, emitter_json_value(emitter_t *emitter, emitter_type_t value_type,
emitter_type_t value_type, const void *value, const void *value) {
const char *table_note_key, emitter_type_t table_note_value_type,
const void *table_note_value) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_json_key_prefix(emitter); emitter_json_key_prefix(emitter);
emitter_printf(emitter, "\"%s\": ", json_key);
emitter_print_value(emitter, emitter_justify_none, -1,
value_type, value);
} else {
emitter_indent(emitter);
emitter_printf(emitter, "%s: ", table_key);
emitter_print_value(emitter, emitter_justify_none, -1, emitter_print_value(emitter, emitter_justify_none, -1,
value_type, value); value_type, value);
if (table_note_key != NULL) { emitter->item_at_depth = true;
emitter_printf(emitter, " (%s: ", table_note_key);
emitter_print_value(emitter, emitter_justify_none, -1,
table_note_value_type, table_note_value);
emitter_printf(emitter, ")");
}
emitter_printf(emitter, "\n");
} }
emitter->item_at_depth = true;
} }
/* Shorthand for calling emitter_json_key and then emitter_json_value. */
static inline void static inline void
emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key, emitter_json_kv(emitter_t *emitter, const char *json_key,
emitter_type_t value_type, const void *value) { emitter_type_t value_type, const void *value) {
emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL, emitter_json_key(emitter, json_key);
emitter_type_bool, NULL); emitter_json_value(emitter, value_type, value);
} }
static inline void static inline void
emitter_json_kv(emitter_t *emitter, const char *json_key, emitter_json_array_begin(emitter_t *emitter) {
emitter_type_t value_type, const void *value) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_json) {
emitter_kv(emitter, json_key, NULL, value_type, value); emitter_json_key_prefix(emitter);
emitter_printf(emitter, "[");
emitter_nest_inc(emitter);
} }
} }
/* Shorthand for calling emitter_json_key and then emitter_json_array_begin. */
static inline void static inline void
emitter_table_kv(emitter_t *emitter, const char *table_key, emitter_json_array_kv_begin(emitter_t *emitter, const char *json_key) {
emitter_type_t value_type, const void *value) { emitter_json_key(emitter, json_key);
if (emitter->output == emitter_output_table) { emitter_json_array_begin(emitter);
emitter_kv(emitter, NULL, table_key, value_type, value); }
static inline void
emitter_json_array_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n");
emitter_indent(emitter);
emitter_printf(emitter, "]");
} }
} }
static inline void static inline void
emitter_dict_begin(emitter_t *emitter, const char *json_key, emitter_json_object_begin(emitter_t *emitter) {
const char *table_header) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter); emitter_json_key_prefix(emitter);
emitter_printf(emitter, "\"%s\": {", json_key); emitter_printf(emitter, "{");
emitter_nest_inc(emitter);
} else {
emitter_indent(emitter);
emitter_printf(emitter, "%s\n", table_header);
emitter_nest_inc(emitter); emitter_nest_inc(emitter);
} }
} }
/* Shorthand for calling emitter_json_key and then emitter_json_object_begin. */
static inline void static inline void
emitter_dict_end(emitter_t *emitter) { emitter_json_object_kv_begin(emitter_t *emitter, const char *json_key) {
emitter_json_key(emitter, json_key);
emitter_json_object_begin(emitter);
}
static inline void
emitter_json_object_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0); assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter); emitter_nest_dec(emitter);
emitter_printf(emitter, "\n"); emitter_printf(emitter, "\n");
emitter_indent(emitter); emitter_indent(emitter);
emitter_printf(emitter, "}"); emitter_printf(emitter, "}");
} else {
emitter_nest_dec(emitter);
} }
} }
/******************************************************************************/
/* Table public API. */
static inline void static inline void
emitter_json_dict_begin(emitter_t *emitter, const char *json_key) { emitter_table_dict_begin(emitter_t *emitter, const char *table_key) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_table) {
emitter_dict_begin(emitter, json_key, NULL); emitter_indent(emitter);
emitter_printf(emitter, "%s\n", table_key);
emitter_nest_inc(emitter);
} }
} }
static inline void static inline void
emitter_json_dict_end(emitter_t *emitter) { emitter_table_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_table) {
emitter_dict_end(emitter); emitter_nest_dec(emitter);
} }
} }
static inline void static inline void
emitter_table_dict_begin(emitter_t *emitter, const char *table_key) { emitter_table_kv_note(emitter_t *emitter, const char *table_key,
emitter_type_t value_type, const void *value,
const char *table_note_key, emitter_type_t table_note_value_type,
const void *table_note_value) {
if (emitter->output == emitter_output_table) { if (emitter->output == emitter_output_table) {
emitter_dict_begin(emitter, NULL, table_key); emitter_indent(emitter);
emitter_printf(emitter, "%s: ", table_key);
emitter_print_value(emitter, emitter_justify_none, -1,
value_type, value);
if (table_note_key != NULL) {
emitter_printf(emitter, " (%s: ", table_note_key);
emitter_print_value(emitter, emitter_justify_none, -1,
table_note_value_type, table_note_value);
emitter_printf(emitter, ")");
}
emitter_printf(emitter, "\n");
} }
emitter->item_at_depth = true;
} }
static inline void static inline void
emitter_table_dict_end(emitter_t *emitter) { emitter_table_kv(emitter_t *emitter, const char *table_key,
emitter_type_t value_type, const void *value) {
emitter_table_kv_note(emitter, table_key, value_type, value, NULL,
emitter_type_bool, NULL);
}
/* Write to the emitter the given string, but only in table mode. */
JEMALLOC_FORMAT_PRINTF(2, 3)
static inline void
emitter_table_printf(emitter_t *emitter, const char *format, ...) {
if (emitter->output == emitter_output_table) { if (emitter->output == emitter_output_table) {
emitter_dict_end(emitter); va_list ap;
va_start(ap, format);
malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap);
va_end(ap);
} }
} }
static inline void static inline void
emitter_json_arr_begin(emitter_t *emitter, const char *json_key) { emitter_table_row(emitter_t *emitter, emitter_row_t *row) {
if (emitter->output == emitter_output_json) { if (emitter->output != emitter_output_table) {
emitter_json_key_prefix(emitter); return;
emitter_printf(emitter, "\"%s\": [", json_key); }
emitter_nest_inc(emitter); emitter_col_t *col;
ql_foreach(col, &row->cols, link) {
emitter_print_value(emitter, col->justify, col->width,
col->type, (const void *)&col->bool_val);
} }
emitter_table_printf(emitter, "\n");
}
static inline void
emitter_row_init(emitter_row_t *row) {
ql_new(&row->cols);
} }
static inline void static inline void
emitter_json_arr_end(emitter_t *emitter) { emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
ql_elm_new(col, link);
ql_tail_insert(&row->cols, col, link);
}
/******************************************************************************/
/*
* Generalized public API. Emits using either JSON or table, according to
* settings in the emitter_t. */
/*
* Note emits a different kv pair as well, but only in table mode. Omits the
* note if table_note_key is NULL.
*/
static inline void
emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key,
emitter_type_t value_type, const void *value,
const char *table_note_key, emitter_type_t table_note_value_type,
const void *table_note_value) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0); emitter_json_key(emitter, json_key);
emitter_nest_dec(emitter); emitter_json_value(emitter, value_type, value);
emitter_printf(emitter, "\n"); } else {
emitter_indent(emitter); emitter_table_kv_note(emitter, table_key, value_type, value,
emitter_printf(emitter, "]"); table_note_key, table_note_value_type, table_note_value);
} }
emitter->item_at_depth = true;
} }
static inline void static inline void
emitter_json_arr_obj_begin(emitter_t *emitter) { emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key,
emitter_type_t value_type, const void *value) {
emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL,
emitter_type_bool, NULL);
}
static inline void
emitter_dict_begin(emitter_t *emitter, const char *json_key,
const char *table_header) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter); emitter_json_key(emitter, json_key);
emitter_printf(emitter, "{"); emitter_json_object_begin(emitter);
emitter_nest_inc(emitter); } else {
emitter_table_dict_begin(emitter, table_header);
} }
} }
static inline void static inline void
emitter_json_arr_obj_end(emitter_t *emitter) { emitter_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0); emitter_json_object_end(emitter);
emitter_nest_dec(emitter); } else {
emitter_printf(emitter, "\n"); emitter_table_dict_end(emitter);
emitter_indent(emitter);
emitter_printf(emitter, "}");
} }
} }
static inline void static inline void
emitter_json_arr_value(emitter_t *emitter, emitter_type_t value_type, emitter_begin(emitter_t *emitter) {
const void *value) {
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter); assert(emitter->nesting_depth == 0);
emitter_print_value(emitter, emitter_justify_none, -1, emitter_printf(emitter, "{");
value_type, value); emitter_nest_inc(emitter);
} else {
/*
* This guarantees that we always call write_cb at least once.
* This is useful if some invariant is established by each call
* to write_cb, but doesn't hold initially: e.g., some buffer
* holds a null-terminated string.
*/
emitter_printf(emitter, "%s", "");
} }
} }
static inline void static inline void
emitter_table_row(emitter_t *emitter, emitter_row_t *row) { emitter_end(emitter_t *emitter) {
if (emitter->output != emitter_output_table) { if (emitter->output == emitter_output_json) {
return; assert(emitter->nesting_depth == 1);
} emitter_nest_dec(emitter);
emitter_col_t *col; emitter_printf(emitter, "\n}\n");
ql_foreach(col, &row->cols, link) {
emitter_print_value(emitter, col->justify, col->width,
col->type, (const void *)&col->bool_val);
} }
emitter_table_printf(emitter, "\n");
} }
#endif /* JEMALLOC_INTERNAL_EMITTER_H */ #endif /* JEMALLOC_INTERNAL_EMITTER_H */
...@@ -24,13 +24,17 @@ size_t extent_size_quantize_floor(size_t size); ...@@ -24,13 +24,17 @@ size_t extent_size_quantize_floor(size_t size);
size_t extent_size_quantize_ceil(size_t size); size_t extent_size_quantize_ceil(size_t size);
#endif #endif
rb_proto(, extent_avail_, extent_tree_t, extent_t) ph_proto(, extent_avail_, extent_tree_t, extent_t)
ph_proto(, extent_heap_, extent_heap_t, extent_t) ph_proto(, extent_heap_, extent_heap_t, extent_t)
bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state,
bool delay_coalesce); bool delay_coalesce);
extent_state_t extents_state_get(const extents_t *extents); extent_state_t extents_state_get(const extents_t *extents);
size_t extents_npages_get(extents_t *extents); size_t extents_npages_get(extents_t *extents);
/* Get the number of extents in the given page size index. */
size_t extents_nextents_get(extents_t *extents, pszind_t ind);
/* Get the sum total bytes of the extents in the given page size index. */
size_t extents_nbytes_get(extents_t *extents, pszind_t ind);
extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr,
size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
...@@ -70,4 +74,10 @@ bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ...@@ -70,4 +74,10 @@ bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
bool extent_boot(void); bool extent_boot(void);
void extent_util_stats_get(tsdn_t *tsdn, const void *ptr,
size_t *nfree, size_t *nregs, size_t *size);
void extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
size_t *nfree, size_t *nregs, size_t *size,
size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr);
#endif /* JEMALLOC_INTERNAL_EXTENT_EXTERNS_H */ #endif /* JEMALLOC_INTERNAL_EXTENT_EXTERNS_H */
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
#include "jemalloc/internal/pages.h" #include "jemalloc/internal/pages.h"
#include "jemalloc/internal/prng.h" #include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ql.h" #include "jemalloc/internal/ql.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h" #include "jemalloc/internal/sz.h"
static inline void static inline void
...@@ -34,18 +35,19 @@ extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { ...@@ -34,18 +35,19 @@ extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) {
(uintptr_t)extent2); (uintptr_t)extent2);
} }
static inline arena_t * static inline unsigned
extent_arena_get(const extent_t *extent) { extent_arena_ind_get(const extent_t *extent) {
unsigned arena_ind = (unsigned)((extent->e_bits & unsigned arena_ind = (unsigned)((extent->e_bits &
EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT);
/*
* The following check is omitted because we should never actually read
* a NULL arena pointer.
*/
if (false && arena_ind >= MALLOCX_ARENA_LIMIT) {
return NULL;
}
assert(arena_ind < MALLOCX_ARENA_LIMIT); assert(arena_ind < MALLOCX_ARENA_LIMIT);
return arena_ind;
}
static inline arena_t *
extent_arena_get(const extent_t *extent) {
unsigned arena_ind = extent_arena_ind_get(extent);
return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE); return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE);
} }
...@@ -53,14 +55,14 @@ static inline szind_t ...@@ -53,14 +55,14 @@ static inline szind_t
extent_szind_get_maybe_invalid(const extent_t *extent) { extent_szind_get_maybe_invalid(const extent_t *extent) {
szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >> szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >>
EXTENT_BITS_SZIND_SHIFT); EXTENT_BITS_SZIND_SHIFT);
assert(szind <= NSIZES); assert(szind <= SC_NSIZES);
return szind; return szind;
} }
static inline szind_t static inline szind_t
extent_szind_get(const extent_t *extent) { extent_szind_get(const extent_t *extent) {
szind_t szind = extent_szind_get_maybe_invalid(extent); szind_t szind = extent_szind_get_maybe_invalid(extent);
assert(szind < NSIZES); /* Never call when "invalid". */ assert(szind < SC_NSIZES); /* Never call when "invalid". */
return szind; return szind;
} }
...@@ -69,6 +71,14 @@ extent_usize_get(const extent_t *extent) { ...@@ -69,6 +71,14 @@ extent_usize_get(const extent_t *extent) {
return sz_index2size(extent_szind_get(extent)); return sz_index2size(extent_szind_get(extent));
} }
static inline unsigned
extent_binshard_get(const extent_t *extent) {
unsigned binshard = (unsigned)((extent->e_bits &
EXTENT_BITS_BINSHARD_MASK) >> EXTENT_BITS_BINSHARD_SHIFT);
assert(binshard < bin_infos[extent_szind_get(extent)].n_shards);
return binshard;
}
static inline size_t static inline size_t
extent_sn_get(const extent_t *extent) { extent_sn_get(const extent_t *extent) {
return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >> return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >>
...@@ -176,6 +186,11 @@ extent_prof_tctx_get(const extent_t *extent) { ...@@ -176,6 +186,11 @@ extent_prof_tctx_get(const extent_t *extent) {
ATOMIC_ACQUIRE); ATOMIC_ACQUIRE);
} }
static inline nstime_t
extent_prof_alloc_time_get(const extent_t *extent) {
return extent->e_alloc_time;
}
static inline void static inline void
extent_arena_set(extent_t *extent, arena_t *arena) { extent_arena_set(extent_t *extent, arena_t *arena) {
unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U << unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U <<
...@@ -184,13 +199,21 @@ extent_arena_set(extent_t *extent, arena_t *arena) { ...@@ -184,13 +199,21 @@ extent_arena_set(extent_t *extent, arena_t *arena) {
((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT); ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT);
} }
static inline void
extent_binshard_set(extent_t *extent, unsigned binshard) {
/* The assertion assumes szind is set already. */
assert(binshard < bin_infos[extent_szind_get(extent)].n_shards);
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_BINSHARD_MASK) |
((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT);
}
static inline void static inline void
extent_addr_set(extent_t *extent, void *addr) { extent_addr_set(extent_t *extent, void *addr) {
extent->e_addr = addr; extent->e_addr = addr;
} }
static inline void static inline void
extent_addr_randomize(UNUSED tsdn_t *tsdn, extent_t *extent, size_t alignment) { extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) {
assert(extent_base_get(extent) == extent_addr_get(extent)); assert(extent_base_get(extent) == extent_addr_get(extent));
if (alignment < PAGE) { if (alignment < PAGE) {
...@@ -234,7 +257,7 @@ extent_bsize_set(extent_t *extent, size_t bsize) { ...@@ -234,7 +257,7 @@ extent_bsize_set(extent_t *extent, size_t bsize) {
static inline void static inline void
extent_szind_set(extent_t *extent, szind_t szind) { extent_szind_set(extent_t *extent, szind_t szind) {
assert(szind <= NSIZES); /* NSIZES means "invalid". */ assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) |
((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT); ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT);
} }
...@@ -246,6 +269,16 @@ extent_nfree_set(extent_t *extent, unsigned nfree) { ...@@ -246,6 +269,16 @@ extent_nfree_set(extent_t *extent, unsigned nfree) {
((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT);
} }
static inline void
extent_nfree_binshard_set(extent_t *extent, unsigned nfree, unsigned binshard) {
/* The assertion assumes szind is set already. */
assert(binshard < bin_infos[extent_szind_get(extent)].n_shards);
extent->e_bits = (extent->e_bits &
(~EXTENT_BITS_NFREE_MASK & ~EXTENT_BITS_BINSHARD_MASK)) |
((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT) |
((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT);
}
static inline void static inline void
extent_nfree_inc(extent_t *extent) { extent_nfree_inc(extent_t *extent) {
assert(extent_slab_get(extent)); assert(extent_slab_get(extent));
...@@ -258,6 +291,12 @@ extent_nfree_dec(extent_t *extent) { ...@@ -258,6 +291,12 @@ extent_nfree_dec(extent_t *extent) {
extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
} }
static inline void
extent_nfree_sub(extent_t *extent, uint64_t n) {
assert(extent_slab_get(extent));
extent->e_bits -= (n << EXTENT_BITS_NFREE_SHIFT);
}
static inline void static inline void
extent_sn_set(extent_t *extent, size_t sn) { extent_sn_set(extent_t *extent, size_t sn) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) |
...@@ -299,10 +338,35 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { ...@@ -299,10 +338,35 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) {
atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE); atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE);
} }
static inline void
extent_prof_alloc_time_set(extent_t *extent, nstime_t t) {
nstime_copy(&extent->e_alloc_time, &t);
}
static inline bool
extent_is_head_get(extent_t *extent) {
if (maps_coalesce) {
not_reached();
}
return (bool)((extent->e_bits & EXTENT_BITS_IS_HEAD_MASK) >>
EXTENT_BITS_IS_HEAD_SHIFT);
}
static inline void
extent_is_head_set(extent_t *extent, bool is_head) {
if (maps_coalesce) {
not_reached();
}
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_IS_HEAD_MASK) |
((uint64_t)is_head << EXTENT_BITS_IS_HEAD_SHIFT);
}
static inline void static inline void
extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed,
bool committed, bool dumpable) { bool committed, bool dumpable, extent_head_state_t is_head) {
assert(addr == PAGE_ADDR2BASE(addr) || !slab); assert(addr == PAGE_ADDR2BASE(addr) || !slab);
extent_arena_set(extent, arena); extent_arena_set(extent, arena);
...@@ -316,6 +380,10 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, ...@@ -316,6 +380,10 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
extent_committed_set(extent, committed); extent_committed_set(extent, committed);
extent_dumpable_set(extent, dumpable); extent_dumpable_set(extent, dumpable);
ql_elm_new(extent, ql_link); ql_elm_new(extent, ql_link);
if (!maps_coalesce) {
extent_is_head_set(extent, (is_head == EXTENT_IS_HEAD) ? true :
false);
}
if (config_prof) { if (config_prof) {
extent_prof_tctx_set(extent, NULL); extent_prof_tctx_set(extent, NULL);
} }
...@@ -327,7 +395,7 @@ extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { ...@@ -327,7 +395,7 @@ extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) {
extent_addr_set(extent, addr); extent_addr_set(extent, addr);
extent_bsize_set(extent, bsize); extent_bsize_set(extent, bsize);
extent_slab_set(extent, false); extent_slab_set(extent, false);
extent_szind_set(extent, NSIZES); extent_szind_set(extent, SC_NSIZES);
extent_sn_set(extent, sn); extent_sn_set(extent, sn);
extent_state_set(extent, extent_state_active); extent_state_set(extent, extent_state_active);
extent_zeroed_set(extent, true); extent_zeroed_set(extent, true);
......
...@@ -2,11 +2,12 @@ ...@@ -2,11 +2,12 @@
#define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H #define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H
#include "jemalloc/internal/atomic.h" #include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/ql.h" #include "jemalloc/internal/ql.h"
#include "jemalloc/internal/ph.h" #include "jemalloc/internal/ph.h"
#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/sc.h"
typedef enum { typedef enum {
extent_state_active = 0, extent_state_active = 0,
...@@ -28,9 +29,10 @@ struct extent_s { ...@@ -28,9 +29,10 @@ struct extent_s {
* t: state * t: state
* i: szind * i: szind
* f: nfree * f: nfree
* s: bin_shard
* n: sn * n: sn
* *
* nnnnnnnn ... nnnnffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa * nnnnnnnn ... nnnnnnss ssssffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa
* *
* arena_ind: Arena from which this extent came, or all 1 bits if * arena_ind: Arena from which this extent came, or all 1 bits if
* unassociated. * unassociated.
...@@ -75,6 +77,8 @@ struct extent_s { ...@@ -75,6 +77,8 @@ struct extent_s {
* *
* nfree: Number of free regions in slab. * nfree: Number of free regions in slab.
* *
* bin_shard: the shard of the bin from which this extent came.
*
* sn: Serial number (potentially non-unique). * sn: Serial number (potentially non-unique).
* *
* Serial numbers may wrap around if !opt_retain, but as long as * Serial numbers may wrap around if !opt_retain, but as long as
...@@ -112,7 +116,7 @@ struct extent_s { ...@@ -112,7 +116,7 @@ struct extent_s {
#define EXTENT_BITS_STATE_SHIFT (EXTENT_BITS_ZEROED_WIDTH + EXTENT_BITS_ZEROED_SHIFT) #define EXTENT_BITS_STATE_SHIFT (EXTENT_BITS_ZEROED_WIDTH + EXTENT_BITS_ZEROED_SHIFT)
#define EXTENT_BITS_STATE_MASK MASK(EXTENT_BITS_STATE_WIDTH, EXTENT_BITS_STATE_SHIFT) #define EXTENT_BITS_STATE_MASK MASK(EXTENT_BITS_STATE_WIDTH, EXTENT_BITS_STATE_SHIFT)
#define EXTENT_BITS_SZIND_WIDTH LG_CEIL_NSIZES #define EXTENT_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES)
#define EXTENT_BITS_SZIND_SHIFT (EXTENT_BITS_STATE_WIDTH + EXTENT_BITS_STATE_SHIFT) #define EXTENT_BITS_SZIND_SHIFT (EXTENT_BITS_STATE_WIDTH + EXTENT_BITS_STATE_SHIFT)
#define EXTENT_BITS_SZIND_MASK MASK(EXTENT_BITS_SZIND_WIDTH, EXTENT_BITS_SZIND_SHIFT) #define EXTENT_BITS_SZIND_MASK MASK(EXTENT_BITS_SZIND_WIDTH, EXTENT_BITS_SZIND_SHIFT)
...@@ -120,7 +124,15 @@ struct extent_s { ...@@ -120,7 +124,15 @@ struct extent_s {
#define EXTENT_BITS_NFREE_SHIFT (EXTENT_BITS_SZIND_WIDTH + EXTENT_BITS_SZIND_SHIFT) #define EXTENT_BITS_NFREE_SHIFT (EXTENT_BITS_SZIND_WIDTH + EXTENT_BITS_SZIND_SHIFT)
#define EXTENT_BITS_NFREE_MASK MASK(EXTENT_BITS_NFREE_WIDTH, EXTENT_BITS_NFREE_SHIFT) #define EXTENT_BITS_NFREE_MASK MASK(EXTENT_BITS_NFREE_WIDTH, EXTENT_BITS_NFREE_SHIFT)
#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT) #define EXTENT_BITS_BINSHARD_WIDTH 6
#define EXTENT_BITS_BINSHARD_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT)
#define EXTENT_BITS_BINSHARD_MASK MASK(EXTENT_BITS_BINSHARD_WIDTH, EXTENT_BITS_BINSHARD_SHIFT)
#define EXTENT_BITS_IS_HEAD_WIDTH 1
#define EXTENT_BITS_IS_HEAD_SHIFT (EXTENT_BITS_BINSHARD_WIDTH + EXTENT_BITS_BINSHARD_SHIFT)
#define EXTENT_BITS_IS_HEAD_MASK MASK(EXTENT_BITS_IS_HEAD_WIDTH, EXTENT_BITS_IS_HEAD_SHIFT)
#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_IS_HEAD_WIDTH + EXTENT_BITS_IS_HEAD_SHIFT)
#define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT) #define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT)
/* Pointer to the extent that this structure is responsible for. */ /* Pointer to the extent that this structure is responsible for. */
...@@ -160,11 +172,13 @@ struct extent_s { ...@@ -160,11 +172,13 @@ struct extent_s {
/* Small region slab metadata. */ /* Small region slab metadata. */
arena_slab_data_t e_slab_data; arena_slab_data_t e_slab_data;
/* /* Profiling data, used for large objects. */
* Profile counters, used for large objects. Points to a struct {
* prof_tctx_t. /* Time when this was allocated. */
*/ nstime_t e_alloc_time;
atomic_p_t e_prof_tctx; /* Points to a prof_tctx_t. */
atomic_p_t e_prof_tctx;
};
}; };
}; };
typedef ql_head(extent_t) extent_list_t; typedef ql_head(extent_t) extent_list_t;
...@@ -180,14 +194,16 @@ struct extents_s { ...@@ -180,14 +194,16 @@ struct extents_s {
* *
* Synchronization: mtx. * Synchronization: mtx.
*/ */
extent_heap_t heaps[NPSIZES+1]; extent_heap_t heaps[SC_NPSIZES + 1];
atomic_zu_t nextents[SC_NPSIZES + 1];
atomic_zu_t nbytes[SC_NPSIZES + 1];
/* /*
* Bitmap for which set bits correspond to non-empty heaps. * Bitmap for which set bits correspond to non-empty heaps.
* *
* Synchronization: mtx. * Synchronization: mtx.
*/ */
bitmap_t bitmap[BITMAP_GROUPS(NPSIZES+1)]; bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)];
/* /*
* LRU of all extents in heaps. * LRU of all extents in heaps.
...@@ -216,4 +232,25 @@ struct extents_s { ...@@ -216,4 +232,25 @@ struct extents_s {
bool delay_coalesce; bool delay_coalesce;
}; };
/*
* The following two structs are for experimental purposes. See
* experimental_utilization_query_ctl and
* experimental_utilization_batch_query_ctl in src/ctl.c.
*/
struct extent_util_stats_s {
size_t nfree;
size_t nregs;
size_t size;
};
struct extent_util_stats_verbose_s {
void *slabcur_addr;
size_t nfree;
size_t nregs;
size_t size;
size_t bin_nfree;
size_t bin_nregs;
};
#endif /* JEMALLOC_INTERNAL_EXTENT_STRUCTS_H */ #endif /* JEMALLOC_INTERNAL_EXTENT_STRUCTS_H */
...@@ -4,9 +4,10 @@ ...@@ -4,9 +4,10 @@
typedef struct extent_s extent_t; typedef struct extent_s extent_t;
typedef struct extents_s extents_t; typedef struct extents_s extents_t;
#define EXTENT_HOOKS_INITIALIZER NULL typedef struct extent_util_stats_s extent_util_stats_t;
typedef struct extent_util_stats_verbose_s extent_util_stats_verbose_t;
#define EXTENT_GROW_MAX_PIND (NPSIZES - 1) #define EXTENT_HOOKS_INITIALIZER NULL
/* /*
* When reuse (and split) an active extent, (1U << opt_lg_extent_max_active_fit) * When reuse (and split) an active extent, (1U << opt_lg_extent_max_active_fit)
...@@ -14,4 +15,9 @@ typedef struct extents_s extents_t; ...@@ -14,4 +15,9 @@ typedef struct extents_s extents_t;
*/ */
#define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6 #define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6
typedef enum {
EXTENT_NOT_HEAD,
EXTENT_IS_HEAD /* Only relevant for Windows && opt.retain. */
} extent_head_state_t;
#endif /* JEMALLOC_INTERNAL_EXTENT_TYPES_H */ #endif /* JEMALLOC_INTERNAL_EXTENT_TYPES_H */
...@@ -104,8 +104,8 @@ hash_x86_32(const void *key, int len, uint32_t seed) { ...@@ -104,8 +104,8 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
uint32_t k1 = 0; uint32_t k1 = 0;
switch (len & 3) { switch (len & 3) {
case 3: k1 ^= tail[2] << 16; case 3: k1 ^= tail[2] << 16; JEMALLOC_FALLTHROUGH
case 2: k1 ^= tail[1] << 8; case 2: k1 ^= tail[1] << 8; JEMALLOC_FALLTHROUGH
case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15); case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15);
k1 *= c2; h1 ^= k1; k1 *= c2; h1 ^= k1;
} }
...@@ -119,7 +119,7 @@ hash_x86_32(const void *key, int len, uint32_t seed) { ...@@ -119,7 +119,7 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
return h1; return h1;
} }
UNUSED static inline void static inline void
hash_x86_128(const void *key, const int len, uint32_t seed, hash_x86_128(const void *key, const int len, uint32_t seed,
uint64_t r_out[2]) { uint64_t r_out[2]) {
const uint8_t * data = (const uint8_t *) key; const uint8_t * data = (const uint8_t *) key;
...@@ -177,28 +177,29 @@ hash_x86_128(const void *key, const int len, uint32_t seed, ...@@ -177,28 +177,29 @@ hash_x86_128(const void *key, const int len, uint32_t seed,
uint32_t k4 = 0; uint32_t k4 = 0;
switch (len & 15) { switch (len & 15) {
case 15: k4 ^= tail[14] << 16; case 15: k4 ^= tail[14] << 16; JEMALLOC_FALLTHROUGH
case 14: k4 ^= tail[13] << 8; case 14: k4 ^= tail[13] << 8; JEMALLOC_FALLTHROUGH
case 13: k4 ^= tail[12] << 0; case 13: k4 ^= tail[12] << 0;
k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
JEMALLOC_FALLTHROUGH
case 12: k3 ^= tail[11] << 24; case 12: k3 ^= tail[11] << 24; JEMALLOC_FALLTHROUGH
case 11: k3 ^= tail[10] << 16; case 11: k3 ^= tail[10] << 16; JEMALLOC_FALLTHROUGH
case 10: k3 ^= tail[ 9] << 8; case 10: k3 ^= tail[ 9] << 8; JEMALLOC_FALLTHROUGH
case 9: k3 ^= tail[ 8] << 0; case 9: k3 ^= tail[ 8] << 0;
k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
JEMALLOC_FALLTHROUGH
case 8: k2 ^= tail[ 7] << 24; case 8: k2 ^= tail[ 7] << 24; JEMALLOC_FALLTHROUGH
case 7: k2 ^= tail[ 6] << 16; case 7: k2 ^= tail[ 6] << 16; JEMALLOC_FALLTHROUGH
case 6: k2 ^= tail[ 5] << 8; case 6: k2 ^= tail[ 5] << 8; JEMALLOC_FALLTHROUGH
case 5: k2 ^= tail[ 4] << 0; case 5: k2 ^= tail[ 4] << 0;
k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2;
JEMALLOC_FALLTHROUGH
case 4: k1 ^= tail[ 3] << 24; case 4: k1 ^= tail[ 3] << 24; JEMALLOC_FALLTHROUGH
case 3: k1 ^= tail[ 2] << 16; case 3: k1 ^= tail[ 2] << 16; JEMALLOC_FALLTHROUGH
case 2: k1 ^= tail[ 1] << 8; case 2: k1 ^= tail[ 1] << 8; JEMALLOC_FALLTHROUGH
case 1: k1 ^= tail[ 0] << 0; case 1: k1 ^= tail[ 0] << 0;
k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1;
JEMALLOC_FALLTHROUGH
} }
} }
...@@ -220,7 +221,7 @@ hash_x86_128(const void *key, const int len, uint32_t seed, ...@@ -220,7 +221,7 @@ hash_x86_128(const void *key, const int len, uint32_t seed,
r_out[1] = (((uint64_t) h4) << 32) | h3; r_out[1] = (((uint64_t) h4) << 32) | h3;
} }
UNUSED static inline void static inline void
hash_x64_128(const void *key, const int len, const uint32_t seed, hash_x64_128(const void *key, const int len, const uint32_t seed,
uint64_t r_out[2]) { uint64_t r_out[2]) {
const uint8_t *data = (const uint8_t *) key; const uint8_t *data = (const uint8_t *) key;
...@@ -260,22 +261,22 @@ hash_x64_128(const void *key, const int len, const uint32_t seed, ...@@ -260,22 +261,22 @@ hash_x64_128(const void *key, const int len, const uint32_t seed,
uint64_t k2 = 0; uint64_t k2 = 0;
switch (len & 15) { switch (len & 15) {
case 15: k2 ^= ((uint64_t)(tail[14])) << 48; /* falls through */ case 15: k2 ^= ((uint64_t)(tail[14])) << 48; JEMALLOC_FALLTHROUGH
case 14: k2 ^= ((uint64_t)(tail[13])) << 40; /* falls through */ case 14: k2 ^= ((uint64_t)(tail[13])) << 40; JEMALLOC_FALLTHROUGH
case 13: k2 ^= ((uint64_t)(tail[12])) << 32; /* falls through */ case 13: k2 ^= ((uint64_t)(tail[12])) << 32; JEMALLOC_FALLTHROUGH
case 12: k2 ^= ((uint64_t)(tail[11])) << 24; /* falls through */ case 12: k2 ^= ((uint64_t)(tail[11])) << 24; JEMALLOC_FALLTHROUGH
case 11: k2 ^= ((uint64_t)(tail[10])) << 16; /* falls through */ case 11: k2 ^= ((uint64_t)(tail[10])) << 16; JEMALLOC_FALLTHROUGH
case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; /* falls through */ case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; JEMALLOC_FALLTHROUGH
case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0; case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0;
k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
/* falls through */ JEMALLOC_FALLTHROUGH
case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; /* falls through */ case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; JEMALLOC_FALLTHROUGH
case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; /* falls through */ case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; JEMALLOC_FALLTHROUGH
case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; /* falls through */ case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; JEMALLOC_FALLTHROUGH
case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; /* falls through */ case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; JEMALLOC_FALLTHROUGH
case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; /* falls through */ case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; JEMALLOC_FALLTHROUGH
case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; /* falls through */ case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; JEMALLOC_FALLTHROUGH
case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; /* falls through */ case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; JEMALLOC_FALLTHROUGH
case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0; case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0;
k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1;
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment