Unverified Commit c4b4b6c0 authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Merge pull request #9623 from yoav-steinberg/upgrade_jemalloc_5.2.1

Upgraded to jemalloc 5.2.1 from 5.1.0.
Cherry picked all relevant fixes (by diffing our 5.1.0 to upstream 5.10 and finding relevant commits).
Details of what was done:

[cherry-picked] fd7d51c3 2021-05-03 Resolve nonsense static analysis warnings (Oran Agra)
[cherry-picked] 448c435b 2020-09-29 Fix compilation warnings in Lua and jemalloc dependencies (#7785) (YoongHM)
[skipped - already in upstream] 9216b96b 2020-09-21 Fix compilation warning in jemalloc's malloc_vsnprintf (#7789) (YoongHM)
[cherry-picked] 88d71f47 2020-05-20 fix a rare active defrag edge case bug leading to stagnation (Oran Agra)
[skipped - already in upstream] 2fec7d9c 2019-05-30 Jemalloc: Avoid blocking on background thread lock for stats.
[cherry-picked] 920158ec 2018-07-11 Active defrag fixes for 32bit builds (again) (Oran Agra)
[cherry-picked] e8099cab 2018-06-26 add defrag hint support into jemalloc 5 (Oran Agra)
[re-done] 4e729fcd 2018-05-24 Generate configure for Jemalloc. (antirez)

Additionally had to do this:
7727cc2 2021-10-10 Fix defrag to support sharded bins in arena (added in v5.2.1) (Yoav Steinberg)

When reviewing please look at all except the first commit which is just replacing 5.1.0 with 5.2.1 sources.
Also I think we should merge this without squashing to preserve the changes we did to to jemalloc.
parents 276b460e 85737e67
......@@ -632,7 +632,6 @@ malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
*/
write_cb = (je_malloc_message != NULL) ? je_malloc_message :
wrtmessage;
cbopaque = NULL;
}
malloc_vsnprintf(buf, sizeof(buf), format, ap);
......
......@@ -46,7 +46,7 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
void
malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
mutex_prof_data_t *data = &mutex->prof_data;
UNUSED nstime_t before = NSTIME_ZERO_INITIALIZER;
nstime_t before = NSTIME_ZERO_INITIALIZER;
if (ncpus == 1) {
goto label_spin_done;
......@@ -55,7 +55,8 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
int cnt = 0, max_cnt = MALLOC_MUTEX_MAX_SPIN;
do {
spin_cpu_spinwait();
if (!malloc_mutex_trylock_final(mutex)) {
if (!atomic_load_b(&mutex->locked, ATOMIC_RELAXED)
&& !malloc_mutex_trylock_final(mutex)) {
data->n_spin_acquired++;
return;
}
......@@ -144,9 +145,7 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
}
# endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
mutex->lock = OS_UNFAIR_LOCK_INIT;
#elif (defined(JEMALLOC_OSSPIN))
mutex->lock = 0;
mutex->lock = OS_UNFAIR_LOCK_INIT;
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
if (postpone_init) {
mutex->postponed_next = postponed_mutexes;
......
......@@ -180,6 +180,35 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
assert(alignment >= PAGE);
assert(ALIGNMENT_ADDR2BASE(addr, alignment) == addr);
#if defined(__FreeBSD__) && defined(MAP_EXCL)
/*
* FreeBSD has mechanisms both to mmap at specific address without
* touching existing mappings, and to mmap with specific alignment.
*/
{
if (os_overcommits) {
*commit = true;
}
int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
int flags = mmap_flags;
if (addr != NULL) {
flags |= MAP_FIXED | MAP_EXCL;
} else {
unsigned alignment_bits = ffs_zu(alignment);
assert(alignment_bits > 1);
flags |= MAP_ALIGNED(alignment_bits - 1);
}
void *ret = mmap(addr, size, prot, flags, -1, 0);
if (ret == MAP_FAILED) {
ret = NULL;
}
return ret;
}
#endif
/*
* Ideally, there would be a way to specify alignment to mmap() (like
* NetBSD has), but in the absence of such a feature, we have to work
......@@ -261,7 +290,7 @@ pages_decommit(void *addr, size_t size) {
bool
pages_purge_lazy(void *addr, size_t size) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
assert(PAGE_CEILING(size) == size);
if (!pages_can_purge_lazy) {
......@@ -391,6 +420,10 @@ os_page_detect(void) {
GetSystemInfo(&si);
return si.dwPageSize;
#elif defined(__FreeBSD__)
/*
* This returns the value obtained from
* the auxv vector, avoiding a syscall.
*/
return getpagesize();
#else
long result = sysconf(_SC_PAGESIZE);
......@@ -544,6 +577,10 @@ init_thp_state(void) {
close(fd);
#endif
if (nread < 0) {
goto label_error;
}
if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) {
init_system_thp_mode = thp_mode_default;
} else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) {
......@@ -588,6 +625,11 @@ pages_boot(void) {
init_thp_state();
#ifdef __FreeBSD__
/*
* FreeBSD doesn't need the check; madvise(2) is known to work.
*/
#else
/* Detect lazy purge runtime support. */
if (pages_can_purge_lazy) {
bool committed = false;
......@@ -601,6 +643,7 @@ pages_boot(void) {
}
os_pages_unmap(madv_free_page, PAGE);
}
#endif
return false;
}
......@@ -7,6 +7,7 @@
#include "jemalloc/internal/hash.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/emitter.h"
/******************************************************************************/
......@@ -23,7 +24,7 @@
*/
#undef _Unwind_Backtrace
#include <unwind.h>
#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook)
#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook)
#endif
/******************************************************************************/
......@@ -38,6 +39,7 @@ bool opt_prof_gdump = false;
bool opt_prof_final = false;
bool opt_prof_leak = false;
bool opt_prof_accum = false;
bool opt_prof_log = false;
char opt_prof_prefix[
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
......@@ -70,6 +72,100 @@ uint64_t prof_interval = 0;
size_t lg_prof_sample;
typedef enum prof_logging_state_e prof_logging_state_t;
enum prof_logging_state_e {
prof_logging_state_stopped,
prof_logging_state_started,
prof_logging_state_dumping
};
/*
* - stopped: log_start never called, or previous log_stop has completed.
* - started: log_start called, log_stop not called yet. Allocations are logged.
* - dumping: log_stop called but not finished; samples are not logged anymore.
*/
prof_logging_state_t prof_logging_state = prof_logging_state_stopped;
#ifdef JEMALLOC_JET
static bool prof_log_dummy = false;
#endif
/* Incremented for every log file that is output. */
static uint64_t log_seq = 0;
static char log_filename[
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
PATH_MAX +
#endif
1];
/* Timestamp for most recent call to log_start(). */
static nstime_t log_start_timestamp = NSTIME_ZERO_INITIALIZER;
/* Increment these when adding to the log_bt and log_thr linked lists. */
static size_t log_bt_index = 0;
static size_t log_thr_index = 0;
/* Linked list node definitions. These are only used in prof.c. */
typedef struct prof_bt_node_s prof_bt_node_t;
struct prof_bt_node_s {
prof_bt_node_t *next;
size_t index;
prof_bt_t bt;
/* Variable size backtrace vector pointed to by bt. */
void *vec[1];
};
typedef struct prof_thr_node_s prof_thr_node_t;
struct prof_thr_node_s {
prof_thr_node_t *next;
size_t index;
uint64_t thr_uid;
/* Variable size based on thr_name_sz. */
char name[1];
};
typedef struct prof_alloc_node_s prof_alloc_node_t;
/* This is output when logging sampled allocations. */
struct prof_alloc_node_s {
prof_alloc_node_t *next;
/* Indices into an array of thread data. */
size_t alloc_thr_ind;
size_t free_thr_ind;
/* Indices into an array of backtraces. */
size_t alloc_bt_ind;
size_t free_bt_ind;
uint64_t alloc_time_ns;
uint64_t free_time_ns;
size_t usize;
};
/*
* Created on the first call to prof_log_start and deleted on prof_log_stop.
* These are the backtraces and threads that have already been logged by an
* allocation.
*/
static bool log_tables_initialized = false;
static ckh_t log_bt_node_set;
static ckh_t log_thr_node_set;
/* Store linked lists for logged data. */
static prof_bt_node_t *log_bt_first = NULL;
static prof_bt_node_t *log_bt_last = NULL;
static prof_thr_node_t *log_thr_first = NULL;
static prof_thr_node_t *log_thr_last = NULL;
static prof_alloc_node_t *log_alloc_first = NULL;
static prof_alloc_node_t *log_alloc_last = NULL;
/* Protects the prof_logging_state and any log_{...} variable. */
static malloc_mutex_t log_mtx;
/*
* Table of mutexes that are shared among gctx's. These are leaf locks, so
* there is no problem with using them for more than one gctx at the same time.
......@@ -145,6 +241,12 @@ static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
bool even_if_attached);
static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name);
/* Hashtable functions for log_bt_node_set and log_thr_node_set. */
static void prof_thr_node_hash(const void *key, size_t r_hash[2]);
static bool prof_thr_node_keycomp(const void *k1, const void *k2);
static void prof_bt_node_hash(const void *key, size_t r_hash[2]);
static bool prof_bt_node_keycomp(const void *k1, const void *k2);
/******************************************************************************/
/* Red-black trees. */
......@@ -242,6 +344,12 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
prof_tctx_t *tctx) {
prof_tctx_set(tsdn, ptr, usize, NULL, tctx);
/* Get the current time and set this in the extent_t. We'll read this
* when free() is called. */
nstime_t t = NSTIME_ZERO_INITIALIZER;
nstime_update(&t);
prof_alloc_time_set(tsdn, ptr, NULL, t);
malloc_mutex_lock(tsdn, tctx->tdata->lock);
tctx->cnts.curobjs++;
tctx->cnts.curbytes += usize;
......@@ -253,14 +361,174 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
malloc_mutex_unlock(tsdn, tctx->tdata->lock);
}
static size_t
prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) {
assert(prof_logging_state == prof_logging_state_started);
malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx);
prof_bt_node_t dummy_node;
dummy_node.bt = *bt;
prof_bt_node_t *node;
/* See if this backtrace is already cached in the table. */
if (ckh_search(&log_bt_node_set, (void *)(&dummy_node),
(void **)(&node), NULL)) {
size_t sz = offsetof(prof_bt_node_t, vec) +
(bt->len * sizeof(void *));
prof_bt_node_t *new_node = (prof_bt_node_t *)
iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL,
true, arena_get(TSDN_NULL, 0, true), true);
if (log_bt_first == NULL) {
log_bt_first = new_node;
log_bt_last = new_node;
} else {
log_bt_last->next = new_node;
log_bt_last = new_node;
}
new_node->next = NULL;
new_node->index = log_bt_index;
/*
* Copy the backtrace: bt is inside a tdata or gctx, which
* might die before prof_log_stop is called.
*/
new_node->bt.len = bt->len;
memcpy(new_node->vec, bt->vec, bt->len * sizeof(void *));
new_node->bt.vec = new_node->vec;
log_bt_index++;
ckh_insert(tsd, &log_bt_node_set, (void *)new_node, NULL);
return new_node->index;
} else {
return node->index;
}
}
static size_t
prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) {
assert(prof_logging_state == prof_logging_state_started);
malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx);
prof_thr_node_t dummy_node;
dummy_node.thr_uid = thr_uid;
prof_thr_node_t *node;
/* See if this thread is already cached in the table. */
if (ckh_search(&log_thr_node_set, (void *)(&dummy_node),
(void **)(&node), NULL)) {
size_t sz = offsetof(prof_thr_node_t, name) + strlen(name) + 1;
prof_thr_node_t *new_node = (prof_thr_node_t *)
iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL,
true, arena_get(TSDN_NULL, 0, true), true);
if (log_thr_first == NULL) {
log_thr_first = new_node;
log_thr_last = new_node;
} else {
log_thr_last->next = new_node;
log_thr_last = new_node;
}
new_node->next = NULL;
new_node->index = log_thr_index;
new_node->thr_uid = thr_uid;
strcpy(new_node->name, name);
log_thr_index++;
ckh_insert(tsd, &log_thr_node_set, (void *)new_node, NULL);
return new_node->index;
} else {
return node->index;
}
}
static void
prof_try_log(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx) {
malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
prof_tdata_t *cons_tdata = prof_tdata_get(tsd, false);
if (cons_tdata == NULL) {
/*
* We decide not to log these allocations. cons_tdata will be
* NULL only when the current thread is in a weird state (e.g.
* it's being destroyed).
*/
return;
}
malloc_mutex_lock(tsd_tsdn(tsd), &log_mtx);
if (prof_logging_state != prof_logging_state_started) {
goto label_done;
}
if (!log_tables_initialized) {
bool err1 = ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS,
prof_bt_node_hash, prof_bt_node_keycomp);
bool err2 = ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS,
prof_thr_node_hash, prof_thr_node_keycomp);
if (err1 || err2) {
goto label_done;
}
log_tables_initialized = true;
}
nstime_t alloc_time = prof_alloc_time_get(tsd_tsdn(tsd), ptr,
(alloc_ctx_t *)NULL);
nstime_t free_time = NSTIME_ZERO_INITIALIZER;
nstime_update(&free_time);
size_t sz = sizeof(prof_alloc_node_t);
prof_alloc_node_t *new_node = (prof_alloc_node_t *)
iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true,
arena_get(TSDN_NULL, 0, true), true);
const char *prod_thr_name = (tctx->tdata->thread_name == NULL)?
"" : tctx->tdata->thread_name;
const char *cons_thr_name = prof_thread_name_get(tsd);
prof_bt_t bt;
/* Initialize the backtrace, using the buffer in tdata to store it. */
bt_init(&bt, cons_tdata->vec);
prof_backtrace(&bt);
prof_bt_t *cons_bt = &bt;
/* We haven't destroyed tctx yet, so gctx should be good to read. */
prof_bt_t *prod_bt = &tctx->gctx->bt;
new_node->next = NULL;
new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid,
prod_thr_name);
new_node->free_thr_ind = prof_log_thr_index(tsd, cons_tdata->thr_uid,
cons_thr_name);
new_node->alloc_bt_ind = prof_log_bt_index(tsd, prod_bt);
new_node->free_bt_ind = prof_log_bt_index(tsd, cons_bt);
new_node->alloc_time_ns = nstime_ns(&alloc_time);
new_node->free_time_ns = nstime_ns(&free_time);
new_node->usize = usize;
if (log_alloc_first == NULL) {
log_alloc_first = new_node;
log_alloc_last = new_node;
} else {
log_alloc_last->next = new_node;
log_alloc_last = new_node;
}
label_done:
malloc_mutex_unlock(tsd_tsdn(tsd), &log_mtx);
}
void
prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) {
prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize,
prof_tctx_t *tctx) {
malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
assert(tctx->cnts.curobjs > 0);
assert(tctx->cnts.curbytes >= usize);
tctx->cnts.curobjs--;
tctx->cnts.curbytes -= usize;
prof_try_log(tsd, ptr, usize, tctx);
if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) {
prof_tctx_destroy(tsd, tctx);
} else {
......@@ -871,15 +1139,12 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
void
prof_sample_threshold_update(prof_tdata_t *tdata) {
#ifdef JEMALLOC_PROF
uint64_t r;
double u;
if (!config_prof) {
return;
}
if (lg_prof_sample == 0) {
tdata->bytes_until_sample = 0;
tsd_bytes_until_sample_set(tsd_fetch(), 0);
return;
}
......@@ -901,11 +1166,16 @@ prof_sample_threshold_update(prof_tdata_t *tdata) {
* pp 500
* (http://luc.devroye.org/rnbookindex.html)
*/
r = prng_lg_range_u64(&tdata->prng_state, 53);
u = (double)r * (1.0/9007199254740992.0L);
tdata->bytes_until_sample = (uint64_t)(log(u) /
uint64_t r = prng_lg_range_u64(&tdata->prng_state, 53);
double u = (double)r * (1.0/9007199254740992.0L);
uint64_t bytes_until_sample = (uint64_t)(log(u) /
log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
+ (uint64_t)1U;
if (bytes_until_sample > SSIZE_MAX) {
bytes_until_sample = SSIZE_MAX;
}
tsd_bytes_until_sample_set(tsd_fetch(), bytes_until_sample);
#endif
}
......@@ -1022,7 +1292,7 @@ prof_dump_write(bool propagate_err, const char *s) {
}
}
if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) {
if (prof_dump_buf_end + slen - i <= PROF_DUMP_BUFSIZE) {
/* Finish writing. */
n = slen - i;
} else {
......@@ -1033,6 +1303,7 @@ prof_dump_write(bool propagate_err, const char *s) {
prof_dump_buf_end += n;
i += n;
}
assert(i == slen);
return false;
}
......@@ -1887,6 +2158,33 @@ prof_bt_keycomp(const void *k1, const void *k2) {
return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
}
static void
prof_bt_node_hash(const void *key, size_t r_hash[2]) {
const prof_bt_node_t *bt_node = (prof_bt_node_t *)key;
prof_bt_hash((void *)(&bt_node->bt), r_hash);
}
static bool
prof_bt_node_keycomp(const void *k1, const void *k2) {
const prof_bt_node_t *bt_node1 = (prof_bt_node_t *)k1;
const prof_bt_node_t *bt_node2 = (prof_bt_node_t *)k2;
return prof_bt_keycomp((void *)(&bt_node1->bt),
(void *)(&bt_node2->bt));
}
static void
prof_thr_node_hash(const void *key, size_t r_hash[2]) {
const prof_thr_node_t *thr_node = (prof_thr_node_t *)key;
hash(&thr_node->thr_uid, sizeof(uint64_t), 0x94122f35U, r_hash);
}
static bool
prof_thr_node_keycomp(const void *k1, const void *k2) {
const prof_thr_node_t *thr_node1 = (prof_thr_node_t *)k1;
const prof_thr_node_t *thr_node2 = (prof_thr_node_t *)k2;
return thr_node1->thr_uid == thr_node2->thr_uid;
}
static uint64_t
prof_thr_uid_alloc(tsdn_t *tsdn) {
uint64_t thr_uid;
......@@ -2119,6 +2417,368 @@ prof_active_set(tsdn_t *tsdn, bool active) {
return prof_active_old;
}
#ifdef JEMALLOC_JET
size_t
prof_log_bt_count(void) {
size_t cnt = 0;
prof_bt_node_t *node = log_bt_first;
while (node != NULL) {
cnt++;
node = node->next;
}
return cnt;
}
size_t
prof_log_alloc_count(void) {
size_t cnt = 0;
prof_alloc_node_t *node = log_alloc_first;
while (node != NULL) {
cnt++;
node = node->next;
}
return cnt;
}
size_t
prof_log_thr_count(void) {
size_t cnt = 0;
prof_thr_node_t *node = log_thr_first;
while (node != NULL) {
cnt++;
node = node->next;
}
return cnt;
}
bool
prof_log_is_logging(void) {
return prof_logging_state == prof_logging_state_started;
}
bool
prof_log_rep_check(void) {
if (prof_logging_state == prof_logging_state_stopped
&& log_tables_initialized) {
return true;
}
if (log_bt_last != NULL && log_bt_last->next != NULL) {
return true;
}
if (log_thr_last != NULL && log_thr_last->next != NULL) {
return true;
}
if (log_alloc_last != NULL && log_alloc_last->next != NULL) {
return true;
}
size_t bt_count = prof_log_bt_count();
size_t thr_count = prof_log_thr_count();
size_t alloc_count = prof_log_alloc_count();
if (prof_logging_state == prof_logging_state_stopped) {
if (bt_count != 0 || thr_count != 0 || alloc_count || 0) {
return true;
}
}
prof_alloc_node_t *node = log_alloc_first;
while (node != NULL) {
if (node->alloc_bt_ind >= bt_count) {
return true;
}
if (node->free_bt_ind >= bt_count) {
return true;
}
if (node->alloc_thr_ind >= thr_count) {
return true;
}
if (node->free_thr_ind >= thr_count) {
return true;
}
if (node->alloc_time_ns > node->free_time_ns) {
return true;
}
node = node->next;
}
return false;
}
void
prof_log_dummy_set(bool new_value) {
prof_log_dummy = new_value;
}
#endif
bool
prof_log_start(tsdn_t *tsdn, const char *filename) {
if (!opt_prof || !prof_booted) {
return true;
}
bool ret = false;
size_t buf_size = PATH_MAX + 1;
malloc_mutex_lock(tsdn, &log_mtx);
if (prof_logging_state != prof_logging_state_stopped) {
ret = true;
} else if (filename == NULL) {
/* Make default name. */
malloc_snprintf(log_filename, buf_size, "%s.%d.%"FMTu64".json",
opt_prof_prefix, prof_getpid(), log_seq);
log_seq++;
prof_logging_state = prof_logging_state_started;
} else if (strlen(filename) >= buf_size) {
ret = true;
} else {
strcpy(log_filename, filename);
prof_logging_state = prof_logging_state_started;
}
if (!ret) {
nstime_update(&log_start_timestamp);
}
malloc_mutex_unlock(tsdn, &log_mtx);
return ret;
}
/* Used as an atexit function to stop logging on exit. */
static void
prof_log_stop_final(void) {
tsd_t *tsd = tsd_fetch();
prof_log_stop(tsd_tsdn(tsd));
}
struct prof_emitter_cb_arg_s {
int fd;
ssize_t ret;
};
static void
prof_emitter_write_cb(void *opaque, const char *to_write) {
struct prof_emitter_cb_arg_s *arg =
(struct prof_emitter_cb_arg_s *)opaque;
size_t bytes = strlen(to_write);
#ifdef JEMALLOC_JET
if (prof_log_dummy) {
return;
}
#endif
arg->ret = write(arg->fd, (void *)to_write, bytes);
}
/*
* prof_log_emit_{...} goes through the appropriate linked list, emitting each
* node to the json and deallocating it.
*/
static void
prof_log_emit_threads(tsd_t *tsd, emitter_t *emitter) {
emitter_json_array_kv_begin(emitter, "threads");
prof_thr_node_t *thr_node = log_thr_first;
prof_thr_node_t *thr_old_node;
while (thr_node != NULL) {
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "thr_uid", emitter_type_uint64,
&thr_node->thr_uid);
char *thr_name = thr_node->name;
emitter_json_kv(emitter, "thr_name", emitter_type_string,
&thr_name);
emitter_json_object_end(emitter);
thr_old_node = thr_node;
thr_node = thr_node->next;
idalloc(tsd, thr_old_node);
}
emitter_json_array_end(emitter);
}
static void
prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) {
emitter_json_array_kv_begin(emitter, "stack_traces");
prof_bt_node_t *bt_node = log_bt_first;
prof_bt_node_t *bt_old_node;
/*
* Calculate how many hex digits we need: twice number of bytes, two for
* "0x", and then one more for terminating '\0'.
*/
char buf[2 * sizeof(intptr_t) + 3];
size_t buf_sz = sizeof(buf);
while (bt_node != NULL) {
emitter_json_array_begin(emitter);
size_t i;
for (i = 0; i < bt_node->bt.len; i++) {
malloc_snprintf(buf, buf_sz, "%p", bt_node->bt.vec[i]);
char *trace_str = buf;
emitter_json_value(emitter, emitter_type_string,
&trace_str);
}
emitter_json_array_end(emitter);
bt_old_node = bt_node;
bt_node = bt_node->next;
idalloc(tsd, bt_old_node);
}
emitter_json_array_end(emitter);
}
static void
prof_log_emit_allocs(tsd_t *tsd, emitter_t *emitter) {
emitter_json_array_kv_begin(emitter, "allocations");
prof_alloc_node_t *alloc_node = log_alloc_first;
prof_alloc_node_t *alloc_old_node;
while (alloc_node != NULL) {
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "alloc_thread", emitter_type_size,
&alloc_node->alloc_thr_ind);
emitter_json_kv(emitter, "free_thread", emitter_type_size,
&alloc_node->free_thr_ind);
emitter_json_kv(emitter, "alloc_trace", emitter_type_size,
&alloc_node->alloc_bt_ind);
emitter_json_kv(emitter, "free_trace", emitter_type_size,
&alloc_node->free_bt_ind);
emitter_json_kv(emitter, "alloc_timestamp",
emitter_type_uint64, &alloc_node->alloc_time_ns);
emitter_json_kv(emitter, "free_timestamp", emitter_type_uint64,
&alloc_node->free_time_ns);
emitter_json_kv(emitter, "usize", emitter_type_uint64,
&alloc_node->usize);
emitter_json_object_end(emitter);
alloc_old_node = alloc_node;
alloc_node = alloc_node->next;
idalloc(tsd, alloc_old_node);
}
emitter_json_array_end(emitter);
}
static void
prof_log_emit_metadata(emitter_t *emitter) {
emitter_json_object_kv_begin(emitter, "info");
nstime_t now = NSTIME_ZERO_INITIALIZER;
nstime_update(&now);
uint64_t ns = nstime_ns(&now) - nstime_ns(&log_start_timestamp);
emitter_json_kv(emitter, "duration", emitter_type_uint64, &ns);
char *vers = JEMALLOC_VERSION;
emitter_json_kv(emitter, "version",
emitter_type_string, &vers);
emitter_json_kv(emitter, "lg_sample_rate",
emitter_type_int, &lg_prof_sample);
int pid = prof_getpid();
emitter_json_kv(emitter, "pid", emitter_type_int, &pid);
emitter_json_object_end(emitter);
}
bool
prof_log_stop(tsdn_t *tsdn) {
if (!opt_prof || !prof_booted) {
return true;
}
tsd_t *tsd = tsdn_tsd(tsdn);
malloc_mutex_lock(tsdn, &log_mtx);
if (prof_logging_state != prof_logging_state_started) {
malloc_mutex_unlock(tsdn, &log_mtx);
return true;
}
/*
* Set the state to dumping. We'll set it to stopped when we're done.
* Since other threads won't be able to start/stop/log when the state is
* dumping, we don't have to hold the lock during the whole method.
*/
prof_logging_state = prof_logging_state_dumping;
malloc_mutex_unlock(tsdn, &log_mtx);
emitter_t emitter;
/* Create a file. */
int fd;
#ifdef JEMALLOC_JET
if (prof_log_dummy) {
fd = 0;
} else {
fd = creat(log_filename, 0644);
}
#else
fd = creat(log_filename, 0644);
#endif
if (fd == -1) {
malloc_printf("<jemalloc>: creat() for log file \"%s\" "
" failed with %d\n", log_filename, errno);
if (opt_abort) {
abort();
}
return true;
}
/* Emit to json. */
struct prof_emitter_cb_arg_s arg;
arg.fd = fd;
emitter_init(&emitter, emitter_output_json, &prof_emitter_write_cb,
(void *)(&arg));
emitter_begin(&emitter);
prof_log_emit_metadata(&emitter);
prof_log_emit_threads(tsd, &emitter);
prof_log_emit_traces(tsd, &emitter);
prof_log_emit_allocs(tsd, &emitter);
emitter_end(&emitter);
/* Reset global state. */
if (log_tables_initialized) {
ckh_delete(tsd, &log_bt_node_set);
ckh_delete(tsd, &log_thr_node_set);
}
log_tables_initialized = false;
log_bt_index = 0;
log_thr_index = 0;
log_bt_first = NULL;
log_bt_last = NULL;
log_thr_first = NULL;
log_thr_last = NULL;
log_alloc_first = NULL;
log_alloc_last = NULL;
malloc_mutex_lock(tsdn, &log_mtx);
prof_logging_state = prof_logging_state_stopped;
malloc_mutex_unlock(tsdn, &log_mtx);
#ifdef JEMALLOC_JET
if (prof_log_dummy) {
return false;
}
#endif
return close(fd);
}
const char *
prof_thread_name_get(tsd_t *tsd) {
prof_tdata_t *tdata;
......@@ -2355,6 +3015,35 @@ prof_boot2(tsd_t *tsd) {
}
}
if (opt_prof_log) {
prof_log_start(tsd_tsdn(tsd), NULL);
}
if (atexit(prof_log_stop_final) != 0) {
malloc_write("<jemalloc>: Error in atexit() "
"for logging\n");
if (opt_abort) {
abort();
}
}
if (malloc_mutex_init(&log_mtx, "prof_log",
WITNESS_RANK_PROF_LOG, malloc_mutex_rank_exclusive)) {
return true;
}
if (ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS,
prof_bt_node_hash, prof_bt_node_keycomp)) {
return true;
}
if (ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS,
prof_thr_node_hash, prof_thr_node_keycomp)) {
return true;
}
log_tables_initialized = true;
gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),
b0get(), PROF_NCTX_LOCKS * sizeof(malloc_mutex_t),
CACHELINE);
......@@ -2382,16 +3071,14 @@ prof_boot2(tsd_t *tsd) {
return true;
}
}
}
#ifdef JEMALLOC_PROF_LIBGCC
/*
* Cause the backtracing machinery to allocate its internal state
* before enabling profiling.
*/
_Unwind_Backtrace(prof_unwind_init_callback, NULL);
/*
* Cause the backtracing machinery to allocate its internal
* state before enabling profiling.
*/
_Unwind_Backtrace(prof_unwind_init_callback, NULL);
#endif
}
prof_booted = true;
return false;
......
......@@ -39,7 +39,7 @@ rtree_node_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) {
/* Nodes are never deleted during normal operation. */
not_reached();
}
UNUSED rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc =
rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc =
rtree_node_dalloc_impl;
static rtree_leaf_elm_t *
......@@ -54,7 +54,7 @@ rtree_leaf_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) {
/* Leaves are never deleted during normal operation. */
not_reached();
}
UNUSED rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc =
rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc =
rtree_leaf_dalloc_impl;
#ifdef JEMALLOC_JET
......
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
static void (*safety_check_abort)(const char *message);
void safety_check_set_abort(void (*abort_fn)(const char *)) {
safety_check_abort = abort_fn;
}
void safety_check_fail(const char *format, ...) {
char buf[MALLOC_PRINTF_BUFSIZE];
va_list ap;
va_start(ap, format);
malloc_vsnprintf(buf, MALLOC_PRINTF_BUFSIZE, format, ap);
va_end(ap);
if (safety_check_abort == NULL) {
malloc_write(buf);
abort();
} else {
safety_check_abort(buf);
}
}
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/pages.h"
#include "jemalloc/internal/sc.h"
/*
* This module computes the size classes used to satisfy allocations. The logic
* here was ported more or less line-by-line from a shell script, and because of
* that is not the most idiomatic C. Eventually we should fix this, but for now
* at least the damage is compartmentalized to this file.
*/
sc_data_t sc_data_global;
static size_t
reg_size_compute(int lg_base, int lg_delta, int ndelta) {
return (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta);
}
/* Returns the number of pages in the slab. */
static int
slab_size(int lg_page, int lg_base, int lg_delta, int ndelta) {
size_t page = (ZU(1) << lg_page);
size_t reg_size = reg_size_compute(lg_base, lg_delta, ndelta);
size_t try_slab_size = page;
size_t try_nregs = try_slab_size / reg_size;
size_t perfect_slab_size = 0;
bool perfect = false;
/*
* This loop continues until we find the least common multiple of the
* page size and size class size. Size classes are all of the form
* base + ndelta * delta == (ndelta + base/ndelta) * delta, which is
* (ndelta + ngroup) * delta. The way we choose slabbing strategies
* means that delta is at most the page size and ndelta < ngroup. So
* the loop executes for at most 2 * ngroup - 1 iterations, which is
* also the bound on the number of pages in a slab chosen by default.
* With the current default settings, this is at most 7.
*/
while (!perfect) {
perfect_slab_size = try_slab_size;
size_t perfect_nregs = try_nregs;
try_slab_size += page;
try_nregs = try_slab_size / reg_size;
if (perfect_slab_size == perfect_nregs * reg_size) {
perfect = true;
}
}
return (int)(perfect_slab_size / page);
}
static void
size_class(
/* Output. */
sc_t *sc,
/* Configuration decisions. */
int lg_max_lookup, int lg_page, int lg_ngroup,
/* Inputs specific to the size class. */
int index, int lg_base, int lg_delta, int ndelta) {
sc->index = index;
sc->lg_base = lg_base;
sc->lg_delta = lg_delta;
sc->ndelta = ndelta;
sc->psz = (reg_size_compute(lg_base, lg_delta, ndelta)
% (ZU(1) << lg_page) == 0);
size_t size = (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta);
if (index == 0) {
assert(!sc->psz);
}
if (size < (ZU(1) << (lg_page + lg_ngroup))) {
sc->bin = true;
sc->pgs = slab_size(lg_page, lg_base, lg_delta, ndelta);
} else {
sc->bin = false;
sc->pgs = 0;
}
if (size <= (ZU(1) << lg_max_lookup)) {
sc->lg_delta_lookup = lg_delta;
} else {
sc->lg_delta_lookup = 0;
}
}
static void
size_classes(
/* Output. */
sc_data_t *sc_data,
/* Determined by the system. */
size_t lg_ptr_size, int lg_quantum,
/* Configuration decisions. */
int lg_tiny_min, int lg_max_lookup, int lg_page, int lg_ngroup) {
int ptr_bits = (1 << lg_ptr_size) * 8;
int ngroup = (1 << lg_ngroup);
int ntiny = 0;
int nlbins = 0;
int lg_tiny_maxclass = (unsigned)-1;
int nbins = 0;
int npsizes = 0;
int index = 0;
int ndelta = 0;
int lg_base = lg_tiny_min;
int lg_delta = lg_base;
/* Outputs that we update as we go. */
size_t lookup_maxclass = 0;
size_t small_maxclass = 0;
int lg_large_minclass = 0;
size_t large_maxclass = 0;
/* Tiny size classes. */
while (lg_base < lg_quantum) {
sc_t *sc = &sc_data->sc[index];
size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
lg_base, lg_delta, ndelta);
if (sc->lg_delta_lookup != 0) {
nlbins = index + 1;
}
if (sc->psz) {
npsizes++;
}
if (sc->bin) {
nbins++;
}
ntiny++;
/* Final written value is correct. */
lg_tiny_maxclass = lg_base;
index++;
lg_delta = lg_base;
lg_base++;
}
/* First non-tiny (pseudo) group. */
if (ntiny != 0) {
sc_t *sc = &sc_data->sc[index];
/*
* See the note in sc.h; the first non-tiny size class has an
* unusual encoding.
*/
lg_base--;
ndelta = 1;
size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
lg_base, lg_delta, ndelta);
index++;
lg_base++;
lg_delta++;
if (sc->psz) {
npsizes++;
}
if (sc->bin) {
nbins++;
}
}
while (ndelta < ngroup) {
sc_t *sc = &sc_data->sc[index];
size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
lg_base, lg_delta, ndelta);
index++;
ndelta++;
if (sc->psz) {
npsizes++;
}
if (sc->bin) {
nbins++;
}
}
/* All remaining groups. */
lg_base = lg_base + lg_ngroup;
while (lg_base < ptr_bits - 1) {
ndelta = 1;
int ndelta_limit;
if (lg_base == ptr_bits - 2) {
ndelta_limit = ngroup - 1;
} else {
ndelta_limit = ngroup;
}
while (ndelta <= ndelta_limit) {
sc_t *sc = &sc_data->sc[index];
size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index,
lg_base, lg_delta, ndelta);
if (sc->lg_delta_lookup != 0) {
nlbins = index + 1;
/* Final written value is correct. */
lookup_maxclass = (ZU(1) << lg_base)
+ (ZU(ndelta) << lg_delta);
}
if (sc->psz) {
npsizes++;
}
if (sc->bin) {
nbins++;
/* Final written value is correct. */
small_maxclass = (ZU(1) << lg_base)
+ (ZU(ndelta) << lg_delta);
if (lg_ngroup > 0) {
lg_large_minclass = lg_base + 1;
} else {
lg_large_minclass = lg_base + 2;
}
}
large_maxclass = (ZU(1) << lg_base)
+ (ZU(ndelta) << lg_delta);
index++;
ndelta++;
}
lg_base++;
lg_delta++;
}
/* Additional outputs. */
int nsizes = index;
unsigned lg_ceil_nsizes = lg_ceil(nsizes);
/* Fill in the output data. */
sc_data->ntiny = ntiny;
sc_data->nlbins = nlbins;
sc_data->nbins = nbins;
sc_data->nsizes = nsizes;
sc_data->lg_ceil_nsizes = lg_ceil_nsizes;
sc_data->npsizes = npsizes;
sc_data->lg_tiny_maxclass = lg_tiny_maxclass;
sc_data->lookup_maxclass = lookup_maxclass;
sc_data->small_maxclass = small_maxclass;
sc_data->lg_large_minclass = lg_large_minclass;
sc_data->large_minclass = (ZU(1) << lg_large_minclass);
sc_data->large_maxclass = large_maxclass;
/*
* We compute these values in two ways:
* - Incrementally, as above.
* - In macros, in sc.h.
* The computation is easier when done incrementally, but putting it in
* a constant makes it available to the fast paths without having to
* touch the extra global cacheline. We assert, however, that the two
* computations are equivalent.
*/
assert(sc_data->npsizes == SC_NPSIZES);
assert(sc_data->lg_tiny_maxclass == SC_LG_TINY_MAXCLASS);
assert(sc_data->small_maxclass == SC_SMALL_MAXCLASS);
assert(sc_data->large_minclass == SC_LARGE_MINCLASS);
assert(sc_data->lg_large_minclass == SC_LG_LARGE_MINCLASS);
assert(sc_data->large_maxclass == SC_LARGE_MAXCLASS);
/*
* In the allocation fastpath, we want to assume that we can
* unconditionally subtract the requested allocation size from
* a ssize_t, and detect passing through 0 correctly. This
* results in optimal generated code. For this to work, the
* maximum allocation size must be less than SSIZE_MAX.
*/
assert(SC_LARGE_MAXCLASS < SSIZE_MAX);
}
void
sc_data_init(sc_data_t *sc_data) {
assert(!sc_data->initialized);
int lg_max_lookup = 12;
size_classes(sc_data, LG_SIZEOF_PTR, LG_QUANTUM, SC_LG_TINY_MIN,
lg_max_lookup, LG_PAGE, 2);
sc_data->initialized = true;
}
static void
sc_data_update_sc_slab_size(sc_t *sc, size_t reg_size, size_t pgs_guess) {
size_t min_pgs = reg_size / PAGE;
if (reg_size % PAGE != 0) {
min_pgs++;
}
/*
* BITMAP_MAXBITS is actually determined by putting the smallest
* possible size-class on one page, so this can never be 0.
*/
size_t max_pgs = BITMAP_MAXBITS * reg_size / PAGE;
assert(min_pgs <= max_pgs);
assert(min_pgs > 0);
assert(max_pgs >= 1);
if (pgs_guess < min_pgs) {
sc->pgs = (int)min_pgs;
} else if (pgs_guess > max_pgs) {
sc->pgs = (int)max_pgs;
} else {
sc->pgs = (int)pgs_guess;
}
}
void
sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, int pgs) {
assert(data->initialized);
for (int i = 0; i < data->nsizes; i++) {
sc_t *sc = &data->sc[i];
if (!sc->bin) {
break;
}
size_t reg_size = reg_size_compute(sc->lg_base, sc->lg_delta,
sc->ndelta);
if (begin <= reg_size && reg_size <= end) {
sc_data_update_sc_slab_size(sc, reg_size, pgs);
}
}
}
void
sc_boot(sc_data_t *data) {
sc_data_init(data);
}
......@@ -52,6 +52,20 @@ char opt_stats_print_opts[stats_print_tot_num_options+1] = "";
/******************************************************************************/
static uint64_t
rate_per_second(uint64_t value, uint64_t uptime_ns) {
uint64_t billion = 1000000000;
if (uptime_ns == 0 || value == 0) {
return 0;
}
if (uptime_ns < billion) {
return value;
} else {
uint64_t uptime_s = uptime_ns / billion;
return value / uptime_s;
}
}
/* Calculate x.yyy and output a string (takes a fixed sized char array). */
static bool
get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) {
......@@ -104,24 +118,26 @@ mutex_stats_init_cols(emitter_row_t *row, const char *table_name,
#define WIDTH_uint32_t 12
#define WIDTH_uint64_t 16
#define OP(counter, counter_type, human) \
#define OP(counter, counter_type, human, derived, base_counter) \
col = &col_##counter_type[k_##counter_type]; \
++k_##counter_type; \
emitter_col_init(col, row); \
col->justify = emitter_justify_right; \
col->width = WIDTH_##counter_type; \
col->width = derived ? 8 : WIDTH_##counter_type; \
col->type = emitter_type_title; \
col->str_val = human;
MUTEX_PROF_COUNTERS
#undef OP
#undef WIDTH_uint32_t
#undef WIDTH_uint64_t
col_uint64_t[mutex_counter_total_wait_time_ps].width = 10;
}
static void
mutex_stats_read_global(const char *name, emitter_col_t *col_name,
emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) {
emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters],
uint64_t uptime) {
char cmd[MUTEX_CTL_STR_MAX_LENGTH];
col_name->str_val = name;
......@@ -129,12 +145,17 @@ mutex_stats_read_global(const char *name, emitter_col_t *col_name,
emitter_col_t *dst;
#define EMITTER_TYPE_uint32_t emitter_type_uint32
#define EMITTER_TYPE_uint64_t emitter_type_uint64
#define OP(counter, counter_type, human) \
#define OP(counter, counter_type, human, derived, base_counter) \
dst = &col_##counter_type[mutex_counter_##counter]; \
dst->type = EMITTER_TYPE_##counter_type; \
gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \
"mutexes", name, #counter); \
CTL_GET(cmd, (counter_type *)&dst->bool_val, counter_type);
if (!derived) { \
gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \
"mutexes", name, #counter); \
CTL_GET(cmd, (counter_type *)&dst->bool_val, counter_type); \
} else { \
emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \
dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \
}
MUTEX_PROF_COUNTERS
#undef OP
#undef EMITTER_TYPE_uint32_t
......@@ -145,7 +166,8 @@ static void
mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind,
const char *name, emitter_col_t *col_name,
emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) {
emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters],
uint64_t uptime) {
char cmd[MUTEX_CTL_STR_MAX_LENGTH];
col_name->str_val = name;
......@@ -153,13 +175,17 @@ mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind,
emitter_col_t *dst;
#define EMITTER_TYPE_uint32_t emitter_type_uint32
#define EMITTER_TYPE_uint64_t emitter_type_uint64
#define OP(counter, counter_type, human) \
#define OP(counter, counter_type, human, derived, base_counter) \
dst = &col_##counter_type[mutex_counter_##counter]; \
dst->type = EMITTER_TYPE_##counter_type; \
gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \
"arenas.0.mutexes", arena_mutex_names[mutex_ind], #counter);\
CTL_M2_GET(cmd, arena_ind, \
(counter_type *)&dst->bool_val, counter_type);
if (!derived) { \
gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \
"arenas.0.mutexes", arena_mutex_names[mutex_ind], #counter);\
CTL_M2_GET(cmd, arena_ind, (counter_type *)&dst->bool_val, counter_type); \
} else { \
emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \
dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \
}
MUTEX_PROF_COUNTERS
#undef OP
#undef EMITTER_TYPE_uint32_t
......@@ -169,19 +195,25 @@ mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind,
static void
mutex_stats_read_arena_bin(unsigned arena_ind, unsigned bin_ind,
emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) {
emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters],
uint64_t uptime) {
char cmd[MUTEX_CTL_STR_MAX_LENGTH];
emitter_col_t *dst;
#define EMITTER_TYPE_uint32_t emitter_type_uint32
#define EMITTER_TYPE_uint64_t emitter_type_uint64
#define OP(counter, counter_type, human) \
#define OP(counter, counter_type, human, derived, base_counter) \
dst = &col_##counter_type[mutex_counter_##counter]; \
dst->type = EMITTER_TYPE_##counter_type; \
gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \
"arenas.0.bins.0","mutex", #counter); \
CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \
(counter_type *)&dst->bool_val, counter_type);
if (!derived) { \
gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \
"arenas.0.bins.0","mutex", #counter); \
CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \
(counter_type *)&dst->bool_val, counter_type); \
} else { \
emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \
dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \
}
MUTEX_PROF_COUNTERS
#undef OP
#undef EMITTER_TYPE_uint32_t
......@@ -204,19 +236,38 @@ mutex_stats_emit(emitter_t *emitter, emitter_row_t *row,
#define EMITTER_TYPE_uint32_t emitter_type_uint32
#define EMITTER_TYPE_uint64_t emitter_type_uint64
#define OP(counter, type, human) \
col = &col_##type[k_##type]; \
++k_##type; \
emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \
(const void *)&col->bool_val);
#define OP(counter, type, human, derived, base_counter) \
if (!derived) { \
col = &col_##type[k_##type]; \
++k_##type; \
emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \
(const void *)&col->bool_val); \
}
MUTEX_PROF_COUNTERS;
#undef OP
#undef EMITTER_TYPE_uint32_t
#undef EMITTER_TYPE_uint64_t
}
#define COL(row_name, column_name, left_or_right, col_width, etype) \
emitter_col_t col_##column_name; \
emitter_col_init(&col_##column_name, &row_name); \
col_##column_name.justify = emitter_justify_##left_or_right; \
col_##column_name.width = col_width; \
col_##column_name.type = emitter_type_##etype;
#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \
COL(row_name, column_name, left_or_right, col_width, etype) \
emitter_col_t header_##column_name; \
emitter_col_init(&header_##column_name, &header_##row_name); \
header_##column_name.justify = emitter_justify_##left_or_right; \
header_##column_name.width = col_width; \
header_##column_name.type = emitter_type_title; \
header_##column_name.str_val = human ? human : #column_name;
static void
stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) {
stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) {
size_t page;
bool in_gap, in_gap_prev;
unsigned nbins, j;
......@@ -230,43 +281,37 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) {
emitter_row_t row;
emitter_row_init(&row);
#define COL(name, left_or_right, col_width, etype) \
emitter_col_t col_##name; \
emitter_col_init(&col_##name, &row); \
col_##name.justify = emitter_justify_##left_or_right; \
col_##name.width = col_width; \
col_##name.type = emitter_type_##etype; \
emitter_col_t header_col_##name; \
emitter_col_init(&header_col_##name, &header_row); \
header_col_##name.justify = emitter_justify_##left_or_right; \
header_col_##name.width = col_width; \
header_col_##name.type = emitter_type_title; \
header_col_##name.str_val = #name;
COL(size, right, 20, size)
COL(ind, right, 4, unsigned)
COL(allocated, right, 13, uint64)
COL(nmalloc, right, 13, uint64)
COL(ndalloc, right, 13, uint64)
COL(nrequests, right, 13, uint64)
COL(curregs, right, 13, size)
COL(curslabs, right, 13, size)
COL(regs, right, 5, unsigned)
COL(pgs, right, 4, size)
COL_HDR(row, size, NULL, right, 20, size)
COL_HDR(row, ind, NULL, right, 4, unsigned)
COL_HDR(row, allocated, NULL, right, 13, uint64)
COL_HDR(row, nmalloc, NULL, right, 13, uint64)
COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64)
COL_HDR(row, ndalloc, NULL, right, 13, uint64)
COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64)
COL_HDR(row, nrequests, NULL, right, 13, uint64)
COL_HDR(row, nrequests_ps, "(#/sec)", right, 10, uint64)
COL_HDR(row, nshards, NULL, right, 9, unsigned)
COL_HDR(row, curregs, NULL, right, 13, size)
COL_HDR(row, curslabs, NULL, right, 13, size)
COL_HDR(row, nonfull_slabs, NULL, right, 15, size)
COL_HDR(row, regs, NULL, right, 5, unsigned)
COL_HDR(row, pgs, NULL, right, 4, size)
/* To buffer a right- and left-justified column. */
COL(justify_spacer, right, 1, title)
COL(util, right, 6, title)
COL(nfills, right, 13, uint64)
COL(nflushes, right, 13, uint64)
COL(nslabs, right, 13, uint64)
COL(nreslabs, right, 13, uint64)
#undef COL
COL_HDR(row, justify_spacer, NULL, right, 1, title)
COL_HDR(row, util, NULL, right, 6, title)
COL_HDR(row, nfills, NULL, right, 13, uint64)
COL_HDR(row, nfills_ps, "(#/sec)", right, 8, uint64)
COL_HDR(row, nflushes, NULL, right, 13, uint64)
COL_HDR(row, nflushes_ps, "(#/sec)", right, 8, uint64)
COL_HDR(row, nslabs, NULL, right, 13, uint64)
COL_HDR(row, nreslabs, NULL, right, 13, uint64)
COL_HDR(row, nreslabs_ps, "(#/sec)", right, 8, uint64)
/* Don't want to actually print the name. */
header_col_justify_spacer.str_val = " ";
header_justify_spacer.str_val = " ";
col_justify_spacer.str_val = " ";
emitter_col_t col_mutex64[mutex_prof_num_uint64_t_counters];
emitter_col_t col_mutex32[mutex_prof_num_uint32_t_counters];
......@@ -284,16 +329,17 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) {
* We print a "bins:" header as part of the table row; we need to adjust
* the header size column to compensate.
*/
header_col_size.width -=5;
header_size.width -=5;
emitter_table_printf(emitter, "bins:");
emitter_table_row(emitter, &header_row);
emitter_json_arr_begin(emitter, "bins");
emitter_json_array_kv_begin(emitter, "bins");
for (j = 0, in_gap = false; j < nbins; j++) {
uint64_t nslabs;
size_t reg_size, slab_size, curregs;
size_t curslabs;
uint32_t nregs;
size_t nonfull_slabs;
uint32_t nregs, nshards;
uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes;
uint64_t nreslabs;
......@@ -310,6 +356,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) {
CTL_M2_GET("arenas.bin.0.size", j, &reg_size, size_t);
CTL_M2_GET("arenas.bin.0.nregs", j, &nregs, uint32_t);
CTL_M2_GET("arenas.bin.0.slab_size", j, &slab_size, size_t);
CTL_M2_GET("arenas.bin.0.nshards", j, &nshards, uint32_t);
CTL_M2_M4_GET("stats.arenas.0.bins.0.nmalloc", i, j, &nmalloc,
uint64_t);
......@@ -327,13 +374,15 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) {
uint64_t);
CTL_M2_M4_GET("stats.arenas.0.bins.0.curslabs", i, j, &curslabs,
size_t);
CTL_M2_M4_GET("stats.arenas.0.bins.0.nonfull_slabs", i, j, &nonfull_slabs,
size_t);
if (mutex) {
mutex_stats_read_arena_bin(i, j, col_mutex64,
col_mutex32);
col_mutex32, uptime);
}
emitter_json_arr_obj_begin(emitter);
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "nmalloc", emitter_type_uint64,
&nmalloc);
emitter_json_kv(emitter, "ndalloc", emitter_type_uint64,
......@@ -350,13 +399,15 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) {
&nreslabs);
emitter_json_kv(emitter, "curslabs", emitter_type_size,
&curslabs);
emitter_json_kv(emitter, "nonfull_slabs", emitter_type_size,
&nonfull_slabs);
if (mutex) {
emitter_json_dict_begin(emitter, "mutex");
emitter_json_object_kv_begin(emitter, "mutex");
mutex_stats_emit(emitter, NULL, col_mutex64,
col_mutex32);
emitter_json_dict_end(emitter);
emitter_json_object_end(emitter);
}
emitter_json_arr_obj_end(emitter);
emitter_json_object_end(emitter);
size_t availregs = nregs * curslabs;
char util[6];
......@@ -381,17 +432,25 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) {
col_ind.unsigned_val = j;
col_allocated.size_val = curregs * reg_size;
col_nmalloc.uint64_val = nmalloc;
col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime);
col_ndalloc.uint64_val = ndalloc;
col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime);
col_nrequests.uint64_val = nrequests;
col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime);
col_nshards.unsigned_val = nshards;
col_curregs.size_val = curregs;
col_curslabs.size_val = curslabs;
col_nonfull_slabs.size_val = nonfull_slabs;
col_regs.unsigned_val = nregs;
col_pgs.size_val = slab_size / page;
col_util.str_val = util;
col_nfills.uint64_val = nfills;
col_nfills_ps.uint64_val = rate_per_second(nfills, uptime);
col_nflushes.uint64_val = nflushes;
col_nflushes_ps.uint64_val = rate_per_second(nflushes, uptime);
col_nslabs.uint64_val = nslabs;
col_nreslabs.uint64_val = nreslabs;
col_nreslabs_ps.uint64_val = rate_per_second(nreslabs, uptime);
/*
* Note that mutex columns were initialized above, if mutex ==
......@@ -400,7 +459,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) {
emitter_table_row(emitter, &row);
}
emitter_json_arr_end(emitter); /* Close "bins". */
emitter_json_array_end(emitter); /* Close "bins". */
if (in_gap) {
emitter_table_printf(emitter, " ---\n");
......@@ -408,7 +467,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) {
}
static void
stats_arena_lextents_print(emitter_t *emitter, unsigned i) {
stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) {
unsigned nbins, nlextents, j;
bool in_gap, in_gap_prev;
......@@ -420,34 +479,22 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i) {
emitter_row_t row;
emitter_row_init(&row);
#define COL(name, left_or_right, col_width, etype) \
emitter_col_t header_##name; \
emitter_col_init(&header_##name, &header_row); \
header_##name.justify = emitter_justify_##left_or_right; \
header_##name.width = col_width; \
header_##name.type = emitter_type_title; \
header_##name.str_val = #name; \
\
emitter_col_t col_##name; \
emitter_col_init(&col_##name, &row); \
col_##name.justify = emitter_justify_##left_or_right; \
col_##name.width = col_width; \
col_##name.type = emitter_type_##etype;
COL(size, right, 20, size)
COL(ind, right, 4, unsigned)
COL(allocated, right, 13, size)
COL(nmalloc, right, 13, uint64)
COL(ndalloc, right, 13, uint64)
COL(nrequests, right, 13, uint64)
COL(curlextents, right, 13, size)
#undef COL
COL_HDR(row, size, NULL, right, 20, size)
COL_HDR(row, ind, NULL, right, 4, unsigned)
COL_HDR(row, allocated, NULL, right, 13, size)
COL_HDR(row, nmalloc, NULL, right, 13, uint64)
COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64)
COL_HDR(row, ndalloc, NULL, right, 13, uint64)
COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64)
COL_HDR(row, nrequests, NULL, right, 13, uint64)
COL_HDR(row, nrequests_ps, "(#/sec)", right, 8, uint64)
COL_HDR(row, curlextents, NULL, right, 13, size)
/* As with bins, we label the large extents table. */
header_size.width -= 6;
emitter_table_printf(emitter, "large:");
emitter_table_row(emitter, &header_row);
emitter_json_arr_begin(emitter, "lextents");
emitter_json_array_kv_begin(emitter, "lextents");
for (j = 0, in_gap = false; j < nlextents; j++) {
uint64_t nmalloc, ndalloc, nrequests;
......@@ -471,31 +518,122 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i) {
CTL_M2_M4_GET("stats.arenas.0.lextents.0.curlextents", i, j,
&curlextents, size_t);
emitter_json_arr_obj_begin(emitter);
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "curlextents", emitter_type_size,
&curlextents);
emitter_json_arr_obj_end(emitter);
emitter_json_object_end(emitter);
col_size.size_val = lextent_size;
col_ind.unsigned_val = nbins + j;
col_allocated.size_val = curlextents * lextent_size;
col_nmalloc.uint64_val = nmalloc;
col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime);
col_ndalloc.uint64_val = ndalloc;
col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime);
col_nrequests.uint64_val = nrequests;
col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime);
col_curlextents.size_val = curlextents;
if (!in_gap) {
emitter_table_row(emitter, &row);
}
}
emitter_json_arr_end(emitter); /* Close "lextents". */
emitter_json_array_end(emitter); /* Close "lextents". */
if (in_gap) {
emitter_table_printf(emitter, " ---\n");
}
}
static void
stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind) {
stats_arena_extents_print(emitter_t *emitter, unsigned i) {
unsigned j;
bool in_gap, in_gap_prev;
emitter_row_t header_row;
emitter_row_init(&header_row);
emitter_row_t row;
emitter_row_init(&row);
COL_HDR(row, size, NULL, right, 20, size)
COL_HDR(row, ind, NULL, right, 4, unsigned)
COL_HDR(row, ndirty, NULL, right, 13, size)
COL_HDR(row, dirty, NULL, right, 13, size)
COL_HDR(row, nmuzzy, NULL, right, 13, size)
COL_HDR(row, muzzy, NULL, right, 13, size)
COL_HDR(row, nretained, NULL, right, 13, size)
COL_HDR(row, retained, NULL, right, 13, size)
COL_HDR(row, ntotal, NULL, right, 13, size)
COL_HDR(row, total, NULL, right, 13, size)
/* Label this section. */
header_size.width -= 8;
emitter_table_printf(emitter, "extents:");
emitter_table_row(emitter, &header_row);
emitter_json_array_kv_begin(emitter, "extents");
in_gap = false;
for (j = 0; j < SC_NPSIZES; j++) {
size_t ndirty, nmuzzy, nretained, total, dirty_bytes,
muzzy_bytes, retained_bytes, total_bytes;
CTL_M2_M4_GET("stats.arenas.0.extents.0.ndirty", i, j,
&ndirty, size_t);
CTL_M2_M4_GET("stats.arenas.0.extents.0.nmuzzy", i, j,
&nmuzzy, size_t);
CTL_M2_M4_GET("stats.arenas.0.extents.0.nretained", i, j,
&nretained, size_t);
CTL_M2_M4_GET("stats.arenas.0.extents.0.dirty_bytes", i, j,
&dirty_bytes, size_t);
CTL_M2_M4_GET("stats.arenas.0.extents.0.muzzy_bytes", i, j,
&muzzy_bytes, size_t);
CTL_M2_M4_GET("stats.arenas.0.extents.0.retained_bytes", i, j,
&retained_bytes, size_t);
total = ndirty + nmuzzy + nretained;
total_bytes = dirty_bytes + muzzy_bytes + retained_bytes;
in_gap_prev = in_gap;
in_gap = (total == 0);
if (in_gap_prev && !in_gap) {
emitter_table_printf(emitter,
" ---\n");
}
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty);
emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy);
emitter_json_kv(emitter, "nretained", emitter_type_size,
&nretained);
emitter_json_kv(emitter, "dirty_bytes", emitter_type_size,
&dirty_bytes);
emitter_json_kv(emitter, "muzzy_bytes", emitter_type_size,
&muzzy_bytes);
emitter_json_kv(emitter, "retained_bytes", emitter_type_size,
&retained_bytes);
emitter_json_object_end(emitter);
col_size.size_val = sz_pind2sz(j);
col_ind.size_val = j;
col_ndirty.size_val = ndirty;
col_dirty.size_val = dirty_bytes;
col_nmuzzy.size_val = nmuzzy;
col_muzzy.size_val = muzzy_bytes;
col_nretained.size_val = nretained;
col_retained.size_val = retained_bytes;
col_ntotal.size_val = total;
col_total.size_val = total_bytes;
if (!in_gap) {
emitter_table_row(emitter, &row);
}
}
emitter_json_array_end(emitter); /* Close "extents". */
if (in_gap) {
emitter_table_printf(emitter, " ---\n");
}
}
static void
stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptime) {
emitter_row_t row;
emitter_col_t col_name;
emitter_col_t col64[mutex_prof_num_uint64_t_counters];
......@@ -504,36 +642,38 @@ stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind) {
emitter_row_init(&row);
mutex_stats_init_cols(&row, "", &col_name, col64, col32);
emitter_json_dict_begin(emitter, "mutexes");
emitter_json_object_kv_begin(emitter, "mutexes");
emitter_table_row(emitter, &row);
for (mutex_prof_arena_ind_t i = 0; i < mutex_prof_num_arena_mutexes;
i++) {
const char *name = arena_mutex_names[i];
emitter_json_dict_begin(emitter, name);
emitter_json_object_kv_begin(emitter, name);
mutex_stats_read_arena(arena_ind, i, name, &col_name, col64,
col32);
col32, uptime);
mutex_stats_emit(emitter, &row, col64, col32);
emitter_json_dict_end(emitter); /* Close the mutex dict. */
emitter_json_object_end(emitter); /* Close the mutex dict. */
}
emitter_json_dict_end(emitter); /* End "mutexes". */
emitter_json_object_end(emitter); /* End "mutexes". */
}
static void
stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
bool mutex) {
bool mutex, bool extents) {
unsigned nthreads;
const char *dss;
ssize_t dirty_decay_ms, muzzy_decay_ms;
size_t page, pactive, pdirty, pmuzzy, mapped, retained;
size_t base, internal, resident, metadata_thp;
size_t base, internal, resident, metadata_thp, extent_avail;
uint64_t dirty_npurge, dirty_nmadvise, dirty_purged;
uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged;
size_t small_allocated;
uint64_t small_nmalloc, small_ndalloc, small_nrequests;
uint64_t small_nmalloc, small_ndalloc, small_nrequests, small_nfills,
small_nflushes;
size_t large_allocated;
uint64_t large_nmalloc, large_ndalloc, large_nrequests;
size_t tcache_bytes;
uint64_t large_nmalloc, large_ndalloc, large_nrequests, large_nfills,
large_nflushes;
size_t tcache_bytes, abandoned_vm;
uint64_t uptime;
CTL_GET("arenas.page", &page, size_t);
......@@ -594,98 +734,74 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
&muzzy_purged);
/* Table-style emission. */
emitter_col_t decay_type;
emitter_col_init(&decay_type, &decay_row);
decay_type.justify = emitter_justify_right;
decay_type.width = 9;
decay_type.type = emitter_type_title;
decay_type.str_val = "decaying:";
emitter_col_t decay_time;
emitter_col_init(&decay_time, &decay_row);
decay_time.justify = emitter_justify_right;
decay_time.width = 6;
decay_time.type = emitter_type_title;
decay_time.str_val = "time";
emitter_col_t decay_npages;
emitter_col_init(&decay_npages, &decay_row);
decay_npages.justify = emitter_justify_right;
decay_npages.width = 13;
decay_npages.type = emitter_type_title;
decay_npages.str_val = "npages";
emitter_col_t decay_sweeps;
emitter_col_init(&decay_sweeps, &decay_row);
decay_sweeps.justify = emitter_justify_right;
decay_sweeps.width = 13;
decay_sweeps.type = emitter_type_title;
decay_sweeps.str_val = "sweeps";
emitter_col_t decay_madvises;
emitter_col_init(&decay_madvises, &decay_row);
decay_madvises.justify = emitter_justify_right;
decay_madvises.width = 13;
decay_madvises.type = emitter_type_title;
decay_madvises.str_val = "madvises";
emitter_col_t decay_purged;
emitter_col_init(&decay_purged, &decay_row);
decay_purged.justify = emitter_justify_right;
decay_purged.width = 13;
decay_purged.type = emitter_type_title;
decay_purged.str_val = "purged";
COL(decay_row, decay_type, right, 9, title);
col_decay_type.str_val = "decaying:";
COL(decay_row, decay_time, right, 6, title);
col_decay_time.str_val = "time";
COL(decay_row, decay_npages, right, 13, title);
col_decay_npages.str_val = "npages";
COL(decay_row, decay_sweeps, right, 13, title);
col_decay_sweeps.str_val = "sweeps";
COL(decay_row, decay_madvises, right, 13, title);
col_decay_madvises.str_val = "madvises";
COL(decay_row, decay_purged, right, 13, title);
col_decay_purged.str_val = "purged";
/* Title row. */
emitter_table_row(emitter, &decay_row);
/* Dirty row. */
decay_type.str_val = "dirty:";
col_decay_type.str_val = "dirty:";
if (dirty_decay_ms >= 0) {
decay_time.type = emitter_type_ssize;
decay_time.ssize_val = dirty_decay_ms;
col_decay_time.type = emitter_type_ssize;
col_decay_time.ssize_val = dirty_decay_ms;
} else {
decay_time.type = emitter_type_title;
decay_time.str_val = "N/A";
col_decay_time.type = emitter_type_title;
col_decay_time.str_val = "N/A";
}
decay_npages.type = emitter_type_size;
decay_npages.size_val = pdirty;
col_decay_npages.type = emitter_type_size;
col_decay_npages.size_val = pdirty;
decay_sweeps.type = emitter_type_uint64;
decay_sweeps.uint64_val = dirty_npurge;
col_decay_sweeps.type = emitter_type_uint64;
col_decay_sweeps.uint64_val = dirty_npurge;
decay_madvises.type = emitter_type_uint64;
decay_madvises.uint64_val = dirty_nmadvise;
col_decay_madvises.type = emitter_type_uint64;
col_decay_madvises.uint64_val = dirty_nmadvise;
decay_purged.type = emitter_type_uint64;
decay_purged.uint64_val = dirty_purged;
col_decay_purged.type = emitter_type_uint64;
col_decay_purged.uint64_val = dirty_purged;
emitter_table_row(emitter, &decay_row);
/* Muzzy row. */
decay_type.str_val = "muzzy:";
col_decay_type.str_val = "muzzy:";
if (muzzy_decay_ms >= 0) {
decay_time.type = emitter_type_ssize;
decay_time.ssize_val = muzzy_decay_ms;
col_decay_time.type = emitter_type_ssize;
col_decay_time.ssize_val = muzzy_decay_ms;
} else {
decay_time.type = emitter_type_title;
decay_time.str_val = "N/A";
col_decay_time.type = emitter_type_title;
col_decay_time.str_val = "N/A";
}
decay_npages.type = emitter_type_size;
decay_npages.size_val = pmuzzy;
col_decay_npages.type = emitter_type_size;
col_decay_npages.size_val = pmuzzy;
decay_sweeps.type = emitter_type_uint64;
decay_sweeps.uint64_val = muzzy_npurge;
col_decay_sweeps.type = emitter_type_uint64;
col_decay_sweeps.uint64_val = muzzy_npurge;
decay_madvises.type = emitter_type_uint64;
decay_madvises.uint64_val = muzzy_nmadvise;
col_decay_madvises.type = emitter_type_uint64;
col_decay_madvises.uint64_val = muzzy_nmadvise;
decay_purged.type = emitter_type_uint64;
decay_purged.uint64_val = muzzy_purged;
col_decay_purged.type = emitter_type_uint64;
col_decay_purged.uint64_val = muzzy_purged;
emitter_table_row(emitter, &decay_row);
......@@ -693,81 +809,119 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
emitter_row_t alloc_count_row;
emitter_row_init(&alloc_count_row);
emitter_col_t alloc_count_title;
emitter_col_init(&alloc_count_title, &alloc_count_row);
alloc_count_title.justify = emitter_justify_left;
alloc_count_title.width = 25;
alloc_count_title.type = emitter_type_title;
alloc_count_title.str_val = "";
emitter_col_t alloc_count_allocated;
emitter_col_init(&alloc_count_allocated, &alloc_count_row);
alloc_count_allocated.justify = emitter_justify_right;
alloc_count_allocated.width = 12;
alloc_count_allocated.type = emitter_type_title;
alloc_count_allocated.str_val = "allocated";
emitter_col_t alloc_count_nmalloc;
emitter_col_init(&alloc_count_nmalloc, &alloc_count_row);
alloc_count_nmalloc.justify = emitter_justify_right;
alloc_count_nmalloc.width = 12;
alloc_count_nmalloc.type = emitter_type_title;
alloc_count_nmalloc.str_val = "nmalloc";
emitter_col_t alloc_count_ndalloc;
emitter_col_init(&alloc_count_ndalloc, &alloc_count_row);
alloc_count_ndalloc.justify = emitter_justify_right;
alloc_count_ndalloc.width = 12;
alloc_count_ndalloc.type = emitter_type_title;
alloc_count_ndalloc.str_val = "ndalloc";
emitter_col_t alloc_count_nrequests;
emitter_col_init(&alloc_count_nrequests, &alloc_count_row);
alloc_count_nrequests.justify = emitter_justify_right;
alloc_count_nrequests.width = 12;
alloc_count_nrequests.type = emitter_type_title;
alloc_count_nrequests.str_val = "nrequests";
COL(alloc_count_row, count_title, left, 21, title);
col_count_title.str_val = "";
COL(alloc_count_row, count_allocated, right, 16, title);
col_count_allocated.str_val = "allocated";
COL(alloc_count_row, count_nmalloc, right, 16, title);
col_count_nmalloc.str_val = "nmalloc";
COL(alloc_count_row, count_nmalloc_ps, right, 8, title);
col_count_nmalloc_ps.str_val = "(#/sec)";
COL(alloc_count_row, count_ndalloc, right, 16, title);
col_count_ndalloc.str_val = "ndalloc";
COL(alloc_count_row, count_ndalloc_ps, right, 8, title);
col_count_ndalloc_ps.str_val = "(#/sec)";
COL(alloc_count_row, count_nrequests, right, 16, title);
col_count_nrequests.str_val = "nrequests";
COL(alloc_count_row, count_nrequests_ps, right, 10, title);
col_count_nrequests_ps.str_val = "(#/sec)";
COL(alloc_count_row, count_nfills, right, 16, title);
col_count_nfills.str_val = "nfill";
COL(alloc_count_row, count_nfills_ps, right, 10, title);
col_count_nfills_ps.str_val = "(#/sec)";
COL(alloc_count_row, count_nflushes, right, 16, title);
col_count_nflushes.str_val = "nflush";
COL(alloc_count_row, count_nflushes_ps, right, 10, title);
col_count_nflushes_ps.str_val = "(#/sec)";
emitter_table_row(emitter, &alloc_count_row);
col_count_nmalloc_ps.type = emitter_type_uint64;
col_count_ndalloc_ps.type = emitter_type_uint64;
col_count_nrequests_ps.type = emitter_type_uint64;
col_count_nfills_ps.type = emitter_type_uint64;
col_count_nflushes_ps.type = emitter_type_uint64;
#define GET_AND_EMIT_ALLOC_STAT(small_or_large, name, valtype) \
CTL_M2_GET("stats.arenas.0." #small_or_large "." #name, i, \
&small_or_large##_##name, valtype##_t); \
emitter_json_kv(emitter, #name, emitter_type_##valtype, \
&small_or_large##_##name); \
alloc_count_##name.type = emitter_type_##valtype; \
alloc_count_##name.valtype##_val = small_or_large##_##name;
col_count_##name.type = emitter_type_##valtype; \
col_count_##name.valtype##_val = small_or_large##_##name;
emitter_json_dict_begin(emitter, "small");
alloc_count_title.str_val = "small:";
emitter_json_object_kv_begin(emitter, "small");
col_count_title.str_val = "small:";
GET_AND_EMIT_ALLOC_STAT(small, allocated, size)
GET_AND_EMIT_ALLOC_STAT(small, nmalloc, uint64)
col_count_nmalloc_ps.uint64_val =
rate_per_second(col_count_nmalloc.uint64_val, uptime);
GET_AND_EMIT_ALLOC_STAT(small, ndalloc, uint64)
col_count_ndalloc_ps.uint64_val =
rate_per_second(col_count_ndalloc.uint64_val, uptime);
GET_AND_EMIT_ALLOC_STAT(small, nrequests, uint64)
col_count_nrequests_ps.uint64_val =
rate_per_second(col_count_nrequests.uint64_val, uptime);
GET_AND_EMIT_ALLOC_STAT(small, nfills, uint64)
col_count_nfills_ps.uint64_val =
rate_per_second(col_count_nfills.uint64_val, uptime);
GET_AND_EMIT_ALLOC_STAT(small, nflushes, uint64)
col_count_nflushes_ps.uint64_val =
rate_per_second(col_count_nflushes.uint64_val, uptime);
emitter_table_row(emitter, &alloc_count_row);
emitter_json_dict_end(emitter); /* Close "small". */
emitter_json_object_end(emitter); /* Close "small". */
emitter_json_dict_begin(emitter, "large");
alloc_count_title.str_val = "large:";
emitter_json_object_kv_begin(emitter, "large");
col_count_title.str_val = "large:";
GET_AND_EMIT_ALLOC_STAT(large, allocated, size)
GET_AND_EMIT_ALLOC_STAT(large, nmalloc, uint64)
col_count_nmalloc_ps.uint64_val =
rate_per_second(col_count_nmalloc.uint64_val, uptime);
GET_AND_EMIT_ALLOC_STAT(large, ndalloc, uint64)
col_count_ndalloc_ps.uint64_val =
rate_per_second(col_count_ndalloc.uint64_val, uptime);
GET_AND_EMIT_ALLOC_STAT(large, nrequests, uint64)
col_count_nrequests_ps.uint64_val =
rate_per_second(col_count_nrequests.uint64_val, uptime);
GET_AND_EMIT_ALLOC_STAT(large, nfills, uint64)
col_count_nfills_ps.uint64_val =
rate_per_second(col_count_nfills.uint64_val, uptime);
GET_AND_EMIT_ALLOC_STAT(large, nflushes, uint64)
col_count_nflushes_ps.uint64_val =
rate_per_second(col_count_nflushes.uint64_val, uptime);
emitter_table_row(emitter, &alloc_count_row);
emitter_json_dict_end(emitter); /* Close "large". */
emitter_json_object_end(emitter); /* Close "large". */
#undef GET_AND_EMIT_ALLOC_STAT
/* Aggregated small + large stats are emitter only in table mode. */
alloc_count_title.str_val = "total:";
alloc_count_allocated.size_val = small_allocated + large_allocated;
alloc_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc;
alloc_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc;
alloc_count_nrequests.uint64_val = small_nrequests + large_nrequests;
col_count_title.str_val = "total:";
col_count_allocated.size_val = small_allocated + large_allocated;
col_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc;
col_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc;
col_count_nrequests.uint64_val = small_nrequests + large_nrequests;
col_count_nfills.uint64_val = small_nfills + large_nfills;
col_count_nflushes.uint64_val = small_nflushes + large_nflushes;
col_count_nmalloc_ps.uint64_val =
rate_per_second(col_count_nmalloc.uint64_val, uptime);
col_count_ndalloc_ps.uint64_val =
rate_per_second(col_count_ndalloc.uint64_val, uptime);
col_count_nrequests_ps.uint64_val =
rate_per_second(col_count_nrequests.uint64_val, uptime);
col_count_nfills_ps.uint64_val =
rate_per_second(col_count_nfills.uint64_val, uptime);
col_count_nflushes_ps.uint64_val =
rate_per_second(col_count_nflushes.uint64_val, uptime);
emitter_table_row(emitter, &alloc_count_row);
emitter_row_t mem_count_row;
......@@ -776,14 +930,14 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
emitter_col_t mem_count_title;
emitter_col_init(&mem_count_title, &mem_count_row);
mem_count_title.justify = emitter_justify_left;
mem_count_title.width = 25;
mem_count_title.width = 21;
mem_count_title.type = emitter_type_title;
mem_count_title.str_val = "";
emitter_col_t mem_count_val;
emitter_col_init(&mem_count_val, &mem_count_row);
mem_count_val.justify = emitter_justify_right;
mem_count_val.width = 12;
mem_count_val.width = 16;
mem_count_val.type = emitter_type_title;
mem_count_val.str_val = "";
......@@ -809,16 +963,21 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
GET_AND_EMIT_MEM_STAT(metadata_thp)
GET_AND_EMIT_MEM_STAT(tcache_bytes)
GET_AND_EMIT_MEM_STAT(resident)
GET_AND_EMIT_MEM_STAT(abandoned_vm)
GET_AND_EMIT_MEM_STAT(extent_avail)
#undef GET_AND_EMIT_MEM_STAT
if (mutex) {
stats_arena_mutexes_print(emitter, i);
stats_arena_mutexes_print(emitter, i, uptime);
}
if (bins) {
stats_arena_bins_print(emitter, mutex, i);
stats_arena_bins_print(emitter, mutex, i, uptime);
}
if (large) {
stats_arena_lextents_print(emitter, i);
stats_arena_lextents_print(emitter, i, uptime);
}
if (extents) {
stats_arena_extents_print(emitter, i);
}
}
......@@ -857,6 +1016,7 @@ stats_general_print(emitter_t *emitter) {
emitter_kv(emitter, "malloc_conf", "config.malloc_conf",
emitter_type_string, &config_malloc_conf);
CONFIG_WRITE_BOOL(opt_safety_checks);
CONFIG_WRITE_BOOL(prof);
CONFIG_WRITE_BOOL(prof_libgcc);
CONFIG_WRITE_BOOL(prof_libunwind);
......@@ -891,6 +1051,8 @@ stats_general_print(emitter_t *emitter) {
#define OPT_WRITE_UNSIGNED(name) \
OPT_WRITE(name, uv, usz, emitter_type_unsigned)
#define OPT_WRITE_SIZE_T(name) \
OPT_WRITE(name, sv, ssz, emitter_type_size)
#define OPT_WRITE_SSIZE_T(name) \
OPT_WRITE(name, ssv, sssz, emitter_type_ssize)
#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \
......@@ -904,15 +1066,17 @@ stats_general_print(emitter_t *emitter) {
OPT_WRITE_BOOL("abort")
OPT_WRITE_BOOL("abort_conf")
OPT_WRITE_BOOL("confirm_conf")
OPT_WRITE_BOOL("retain")
OPT_WRITE_CHAR_P("dss")
OPT_WRITE_UNSIGNED("narenas")
OPT_WRITE_CHAR_P("percpu_arena")
OPT_WRITE_SIZE_T("oversize_threshold")
OPT_WRITE_CHAR_P("metadata_thp")
OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread")
OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms")
OPT_WRITE_SSIZE_T_MUTABLE("muzzy_decay_ms", "arenas.muzzy_decay_ms")
OPT_WRITE_UNSIGNED("lg_extent_max_active_fit")
OPT_WRITE_SIZE_T("lg_extent_max_active_fit")
OPT_WRITE_CHAR_P("junk")
OPT_WRITE_BOOL("zero")
OPT_WRITE_BOOL("utrace")
......@@ -977,7 +1141,7 @@ stats_general_print(emitter_t *emitter) {
* The json output sticks arena info into an "arenas" dict; the table
* output puts them at the top-level.
*/
emitter_json_dict_begin(emitter, "arenas");
emitter_json_object_kv_begin(emitter, "arenas");
CTL_GET("arenas.narenas", &uv, unsigned);
emitter_kv(emitter, "narenas", "Arenas", emitter_type_unsigned, &uv);
......@@ -1018,9 +1182,9 @@ stats_general_print(emitter_t *emitter) {
* (not just omit the printing).
*/
if (emitter->output == emitter_output_json) {
emitter_json_arr_begin(emitter, "bin");
emitter_json_array_kv_begin(emitter, "bin");
for (unsigned i = 0; i < nbins; i++) {
emitter_json_arr_obj_begin(emitter);
emitter_json_object_begin(emitter);
CTL_M2_GET("arenas.bin.0.size", i, &sv, size_t);
emitter_json_kv(emitter, "size", emitter_type_size,
......@@ -1034,9 +1198,13 @@ stats_general_print(emitter_t *emitter) {
emitter_json_kv(emitter, "slab_size", emitter_type_size,
&sv);
emitter_json_arr_obj_end(emitter);
CTL_M2_GET("arenas.bin.0.nshards", i, &u32v, uint32_t);
emitter_json_kv(emitter, "nshards", emitter_type_uint32,
&u32v);
emitter_json_object_end(emitter);
}
emitter_json_arr_end(emitter); /* Close "bin". */
emitter_json_array_end(emitter); /* Close "bin". */
}
unsigned nlextents;
......@@ -1045,25 +1213,25 @@ stats_general_print(emitter_t *emitter) {
emitter_type_unsigned, &nlextents);
if (emitter->output == emitter_output_json) {
emitter_json_arr_begin(emitter, "lextent");
emitter_json_array_kv_begin(emitter, "lextent");
for (unsigned i = 0; i < nlextents; i++) {
emitter_json_arr_obj_begin(emitter);
emitter_json_object_begin(emitter);
CTL_M2_GET("arenas.lextent.0.size", i, &sv, size_t);
emitter_json_kv(emitter, "size", emitter_type_size,
&sv);
emitter_json_arr_obj_end(emitter);
emitter_json_object_end(emitter);
}
emitter_json_arr_end(emitter); /* Close "lextent". */
emitter_json_array_end(emitter); /* Close "lextent". */
}
emitter_json_dict_end(emitter); /* Close "arenas" */
emitter_json_object_end(emitter); /* Close "arenas" */
}
static void
stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
bool unmerged, bool bins, bool large, bool mutex) {
bool unmerged, bool bins, bool large, bool mutex, bool extents) {
/*
* These should be deleted. We keep them around for a while, to aid in
* the transition to the emitter code.
......@@ -1095,7 +1263,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
}
/* Generic global stats. */
emitter_json_dict_begin(emitter, "stats");
emitter_json_object_kv_begin(emitter, "stats");
emitter_json_kv(emitter, "allocated", emitter_type_size, &allocated);
emitter_json_kv(emitter, "active", emitter_type_size, &active);
emitter_json_kv(emitter, "metadata", emitter_type_size, &metadata);
......@@ -1111,14 +1279,14 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
resident, mapped, retained);
/* Background thread stats. */
emitter_json_dict_begin(emitter, "background_thread");
emitter_json_object_kv_begin(emitter, "background_thread");
emitter_json_kv(emitter, "num_threads", emitter_type_size,
&num_background_threads);
emitter_json_kv(emitter, "num_runs", emitter_type_uint64,
&background_thread_num_runs);
emitter_json_kv(emitter, "run_interval", emitter_type_uint64,
&background_thread_run_interval);
emitter_json_dict_end(emitter); /* Close "background_thread". */
emitter_json_object_end(emitter); /* Close "background_thread". */
emitter_table_printf(emitter, "Background threads: %zu, "
"num_runs: %"FMTu64", run_interval: %"FMTu64" ns\n",
......@@ -1130,30 +1298,33 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
emitter_col_t name;
emitter_col_t col64[mutex_prof_num_uint64_t_counters];
emitter_col_t col32[mutex_prof_num_uint32_t_counters];
uint64_t uptime;
emitter_row_init(&row);
mutex_stats_init_cols(&row, "", &name, col64, col32);
emitter_table_row(emitter, &row);
emitter_json_dict_begin(emitter, "mutexes");
emitter_json_object_kv_begin(emitter, "mutexes");
CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t);
for (int i = 0; i < mutex_prof_num_global_mutexes; i++) {
mutex_stats_read_global(global_mutex_names[i], &name,
col64, col32);
emitter_json_dict_begin(emitter, global_mutex_names[i]);
col64, col32, uptime);
emitter_json_object_kv_begin(emitter, global_mutex_names[i]);
mutex_stats_emit(emitter, &row, col64, col32);
emitter_json_dict_end(emitter);
emitter_json_object_end(emitter);
}
emitter_json_dict_end(emitter); /* Close "mutexes". */
emitter_json_object_end(emitter); /* Close "mutexes". */
}
emitter_json_dict_end(emitter); /* Close "stats". */
emitter_json_object_end(emitter); /* Close "stats". */
if (merged || destroyed || unmerged) {
unsigned narenas;
emitter_json_dict_begin(emitter, "stats.arenas");
emitter_json_object_kv_begin(emitter, "stats.arenas");
CTL_GET("arenas.narenas", &narenas, unsigned);
size_t mib[3];
......@@ -1182,10 +1353,10 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
if (merged && (ninitialized > 1 || !unmerged)) {
/* Print merged arena stats. */
emitter_table_printf(emitter, "Merged arenas stats:\n");
emitter_json_dict_begin(emitter, "merged");
emitter_json_object_kv_begin(emitter, "merged");
stats_arena_print(emitter, MALLCTL_ARENAS_ALL, bins,
large, mutex);
emitter_json_dict_end(emitter); /* Close "merged". */
large, mutex, extents);
emitter_json_object_end(emitter); /* Close "merged". */
}
/* Destroyed stats. */
......@@ -1193,10 +1364,10 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
/* Print destroyed arena stats. */
emitter_table_printf(emitter,
"Destroyed arenas stats:\n");
emitter_json_dict_begin(emitter, "destroyed");
emitter_json_object_kv_begin(emitter, "destroyed");
stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED,
bins, large, mutex);
emitter_json_dict_end(emitter); /* Close "destroyed". */
bins, large, mutex, extents);
emitter_json_object_end(emitter); /* Close "destroyed". */
}
/* Unmerged stats. */
......@@ -1206,18 +1377,18 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
char arena_ind_str[20];
malloc_snprintf(arena_ind_str,
sizeof(arena_ind_str), "%u", i);
emitter_json_dict_begin(emitter,
emitter_json_object_kv_begin(emitter,
arena_ind_str);
emitter_table_printf(emitter,
"arenas[%s]:\n", arena_ind_str);
stats_arena_print(emitter, i, bins,
large, mutex);
large, mutex, extents);
/* Close "<arena-ind>". */
emitter_json_dict_end(emitter);
emitter_json_object_end(emitter);
}
}
}
emitter_json_dict_end(emitter); /* Close "stats.arenas". */
emitter_json_object_end(emitter); /* Close "stats.arenas". */
}
}
......@@ -1270,17 +1441,17 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
cbopaque);
emitter_begin(&emitter);
emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n");
emitter_json_dict_begin(&emitter, "jemalloc");
emitter_json_object_kv_begin(&emitter, "jemalloc");
if (general) {
stats_general_print(&emitter);
}
if (config_stats) {
stats_print_helper(&emitter, merged, destroyed, unmerged,
bins, large, mutex);
bins, large, mutex, extents);
}
emitter_json_dict_end(&emitter); /* Closes the "jemalloc" dict. */
emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */
emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n");
emitter_end(&emitter);
}
......@@ -2,106 +2,63 @@
#include "jemalloc/internal/sz.h"
JEMALLOC_ALIGNED(CACHELINE)
const size_t sz_pind2sz_tab[NPSIZES+1] = {
#define PSZ_yes(lg_grp, ndelta, lg_delta) \
(((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))),
#define PSZ_no(lg_grp, ndelta, lg_delta)
#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \
PSZ_##psz(lg_grp, ndelta, lg_delta)
SIZE_CLASSES
#undef PSZ_yes
#undef PSZ_no
#undef SC
(LARGE_MAXCLASS + PAGE)
};
size_t sz_pind2sz_tab[SC_NPSIZES+1];
static void
sz_boot_pind2sz_tab(const sc_data_t *sc_data) {
int pind = 0;
for (unsigned i = 0; i < SC_NSIZES; i++) {
const sc_t *sc = &sc_data->sc[i];
if (sc->psz) {
sz_pind2sz_tab[pind] = (ZU(1) << sc->lg_base)
+ (ZU(sc->ndelta) << sc->lg_delta);
pind++;
}
}
for (int i = pind; i <= (int)SC_NPSIZES; i++) {
sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE;
}
}
JEMALLOC_ALIGNED(CACHELINE)
const size_t sz_index2size_tab[NSIZES] = {
#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \
((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)),
SIZE_CLASSES
#undef SC
};
size_t sz_index2size_tab[SC_NSIZES];
static void
sz_boot_index2size_tab(const sc_data_t *sc_data) {
for (unsigned i = 0; i < SC_NSIZES; i++) {
const sc_t *sc = &sc_data->sc[i];
sz_index2size_tab[i] = (ZU(1) << sc->lg_base)
+ (ZU(sc->ndelta) << (sc->lg_delta));
}
}
/*
* To keep this table small, we divide sizes by the tiny min size, which gives
* the smallest interval for which the result can change.
*/
JEMALLOC_ALIGNED(CACHELINE)
const uint8_t sz_size2index_tab[] = {
#if LG_TINY_MIN == 0
/* The div module doesn't support division by 1. */
#error "Unsupported LG_TINY_MIN"
#define S2B_0(i) i,
#elif LG_TINY_MIN == 1
#warning "Dangerous LG_TINY_MIN"
#define S2B_1(i) i,
#elif LG_TINY_MIN == 2
#warning "Dangerous LG_TINY_MIN"
#define S2B_2(i) i,
#elif LG_TINY_MIN == 3
#define S2B_3(i) i,
#elif LG_TINY_MIN == 4
#define S2B_4(i) i,
#elif LG_TINY_MIN == 5
#define S2B_5(i) i,
#elif LG_TINY_MIN == 6
#define S2B_6(i) i,
#elif LG_TINY_MIN == 7
#define S2B_7(i) i,
#elif LG_TINY_MIN == 8
#define S2B_8(i) i,
#elif LG_TINY_MIN == 9
#define S2B_9(i) i,
#elif LG_TINY_MIN == 10
#define S2B_10(i) i,
#elif LG_TINY_MIN == 11
#define S2B_11(i) i,
#else
#error "Unsupported LG_TINY_MIN"
#endif
#if LG_TINY_MIN < 1
#define S2B_1(i) S2B_0(i) S2B_0(i)
#endif
#if LG_TINY_MIN < 2
#define S2B_2(i) S2B_1(i) S2B_1(i)
#endif
#if LG_TINY_MIN < 3
#define S2B_3(i) S2B_2(i) S2B_2(i)
#endif
#if LG_TINY_MIN < 4
#define S2B_4(i) S2B_3(i) S2B_3(i)
#endif
#if LG_TINY_MIN < 5
#define S2B_5(i) S2B_4(i) S2B_4(i)
#endif
#if LG_TINY_MIN < 6
#define S2B_6(i) S2B_5(i) S2B_5(i)
#endif
#if LG_TINY_MIN < 7
#define S2B_7(i) S2B_6(i) S2B_6(i)
#endif
#if LG_TINY_MIN < 8
#define S2B_8(i) S2B_7(i) S2B_7(i)
#endif
#if LG_TINY_MIN < 9
#define S2B_9(i) S2B_8(i) S2B_8(i)
#endif
#if LG_TINY_MIN < 10
#define S2B_10(i) S2B_9(i) S2B_9(i)
#endif
#if LG_TINY_MIN < 11
#define S2B_11(i) S2B_10(i) S2B_10(i)
#endif
#define S2B_no(i)
#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \
S2B_##lg_delta_lookup(index)
SIZE_CLASSES
#undef S2B_3
#undef S2B_4
#undef S2B_5
#undef S2B_6
#undef S2B_7
#undef S2B_8
#undef S2B_9
#undef S2B_10
#undef S2B_11
#undef S2B_no
#undef SC
};
uint8_t sz_size2index_tab[(SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1];
static void
sz_boot_size2index_tab(const sc_data_t *sc_data) {
size_t dst_max = (SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1;
size_t dst_ind = 0;
for (unsigned sc_ind = 0; sc_ind < SC_NSIZES && dst_ind < dst_max;
sc_ind++) {
const sc_t *sc = &sc_data->sc[sc_ind];
size_t sz = (ZU(1) << sc->lg_base)
+ (ZU(sc->ndelta) << sc->lg_delta);
size_t max_ind = ((sz + (ZU(1) << SC_LG_TINY_MIN) - 1)
>> SC_LG_TINY_MIN);
for (; dst_ind <= max_ind && dst_ind < dst_max; dst_ind++) {
sz_size2index_tab[dst_ind] = sc_ind;
}
}
}
void
sz_boot(const sc_data_t *sc_data) {
sz_boot_pind2sz_tab(sc_data);
sz_boot_index2size_tab(sc_data);
sz_boot_size2index_tab(sc_data);
}
......@@ -4,7 +4,8 @@
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sc.h"
/******************************************************************************/
/* Data. */
......@@ -41,7 +42,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
szind_t binind = tcache->next_gc_bin;
cache_bin_t *tbin;
if (binind < NBINS) {
if (binind < SC_NBINS) {
tbin = tcache_small_bin_get(tcache, binind);
} else {
tbin = tcache_large_bin_get(tcache, binind);
......@@ -50,7 +51,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
/*
* Flush (ceiling) 3/4 of the objects below the low water mark.
*/
if (binind < NBINS) {
if (binind < SC_NBINS) {
tcache_bin_flush_small(tsd, tcache, tbin, binind,
tbin->ncached - tbin->low_water + (tbin->low_water
>> 2));
......@@ -72,7 +73,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
* Increase fill count by 2X for small bins. Make sure
* lg_fill_div stays greater than 0.
*/
if (binind < NBINS && tcache->lg_fill_div[binind] > 1) {
if (binind < SC_NBINS && tcache->lg_fill_div[binind] > 1) {
tcache->lg_fill_div[binind]--;
}
}
......@@ -100,28 +101,67 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
return ret;
}
/* Enabled with --enable-extra-size-check. */
static void
tbin_extents_lookup_size_check(tsdn_t *tsdn, cache_bin_t *tbin, szind_t binind,
size_t nflush, extent_t **extents){
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
/*
* Verify that the items in the tcache all have the correct size; this
* is useful for catching sized deallocation bugs, also to fail early
* instead of corrupting metadata. Since this can be turned on for opt
* builds, avoid the branch in the loop.
*/
szind_t szind;
size_t sz_sum = binind * nflush;
for (unsigned i = 0 ; i < nflush; i++) {
rtree_extent_szind_read(tsdn, &extents_rtree,
rtree_ctx, (uintptr_t)*(tbin->avail - 1 - i), true,
&extents[i], &szind);
sz_sum -= szind;
}
if (sz_sum != 0) {
safety_check_fail("<jemalloc>: size mismatch in thread cache "
"detected, likely caused by sized deallocation bugs by "
"application. Abort.\n");
abort();
}
}
void
tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
szind_t binind, unsigned rem) {
bool merged_stats = false;
assert(binind < NBINS);
assert(binind < SC_NBINS);
assert((cache_bin_sz_t)rem <= tbin->ncached);
arena_t *arena = tcache->arena;
assert(arena != NULL);
unsigned nflush = tbin->ncached - rem;
VARIABLE_ARRAY(extent_t *, item_extent, nflush);
/* Look up extent once per item. */
for (unsigned i = 0 ; i < nflush; i++) {
item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i));
if (config_opt_safety_checks) {
tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind,
nflush, item_extent);
} else {
for (unsigned i = 0 ; i < nflush; i++) {
item_extent[i] = iealloc(tsd_tsdn(tsd),
*(tbin->avail - 1 - i));
}
}
while (nflush > 0) {
/* Lock the arena bin associated with the first object. */
extent_t *extent = item_extent[0];
arena_t *bin_arena = extent_arena_get(extent);
bin_t *bin = &bin_arena->bins[binind];
unsigned bin_arena_ind = extent_arena_ind_get(extent);
arena_t *bin_arena = arena_get(tsd_tsdn(tsd), bin_arena_ind,
false);
unsigned binshard = extent_binshard_get(extent);
assert(binshard < bin_infos[binind].n_shards);
bin_t *bin = &bin_arena->bins[binind].bin_shards[binshard];
if (config_prof && bin_arena == arena) {
if (arena_prof_accum(tsd_tsdn(tsd), arena,
......@@ -132,8 +172,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
}
malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
if (config_stats && bin_arena == arena) {
assert(!merged_stats);
if (config_stats && bin_arena == arena && !merged_stats) {
merged_stats = true;
bin->stats.nflushes++;
bin->stats.nrequests += tbin->tstats.nrequests;
......@@ -145,9 +184,10 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
extent = item_extent[i];
assert(ptr != NULL && extent != NULL);
if (extent_arena_get(extent) == bin_arena) {
if (extent_arena_ind_get(extent) == bin_arena_ind
&& extent_binshard_get(extent) == binshard) {
arena_dalloc_bin_junked_locked(tsd_tsdn(tsd),
bin_arena, extent, ptr);
bin_arena, bin, binind, extent, ptr);
} else {
/*
* This object was allocated via a different
......@@ -169,8 +209,9 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
* The flush loop didn't happen to flush to this thread's
* arena, so the stats didn't get merged. Manually do so now.
*/
bin_t *bin = &arena->bins[binind];
malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
unsigned binshard;
bin_t *bin = arena_bin_choose_lock(tsd_tsdn(tsd), arena, binind,
&binshard);
bin->stats.nflushes++;
bin->stats.nrequests += tbin->tstats.nrequests;
tbin->tstats.nrequests = 0;
......@@ -193,50 +234,63 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind,
assert(binind < nhbins);
assert((cache_bin_sz_t)rem <= tbin->ncached);
arena_t *arena = tcache->arena;
assert(arena != NULL);
arena_t *tcache_arena = tcache->arena;
assert(tcache_arena != NULL);
unsigned nflush = tbin->ncached - rem;
VARIABLE_ARRAY(extent_t *, item_extent, nflush);
#ifndef JEMALLOC_EXTRA_SIZE_CHECK
/* Look up extent once per item. */
for (unsigned i = 0 ; i < nflush; i++) {
item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i));
}
#else
tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush,
item_extent);
#endif
while (nflush > 0) {
/* Lock the arena associated with the first object. */
extent_t *extent = item_extent[0];
arena_t *locked_arena = extent_arena_get(extent);
UNUSED bool idump;
unsigned locked_arena_ind = extent_arena_ind_get(extent);
arena_t *locked_arena = arena_get(tsd_tsdn(tsd),
locked_arena_ind, false);
bool idump;
if (config_prof) {
idump = false;
}
malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx);
bool lock_large = !arena_is_auto(locked_arena);
if (lock_large) {
malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx);
}
for (unsigned i = 0; i < nflush; i++) {
void *ptr = *(tbin->avail - 1 - i);
assert(ptr != NULL);
extent = item_extent[i];
if (extent_arena_get(extent) == locked_arena) {
if (extent_arena_ind_get(extent) == locked_arena_ind) {
large_dalloc_prep_junked_locked(tsd_tsdn(tsd),
extent);
}
}
if ((config_prof || config_stats) && locked_arena == arena) {
if ((config_prof || config_stats) &&
(locked_arena == tcache_arena)) {
if (config_prof) {
idump = arena_prof_accum(tsd_tsdn(tsd), arena,
tcache->prof_accumbytes);
idump = arena_prof_accum(tsd_tsdn(tsd),
tcache_arena, tcache->prof_accumbytes);
tcache->prof_accumbytes = 0;
}
if (config_stats) {
merged_stats = true;
arena_stats_large_nrequests_add(tsd_tsdn(tsd),
&arena->stats, binind,
arena_stats_large_flush_nrequests_add(
tsd_tsdn(tsd), &tcache_arena->stats, binind,
tbin->tstats.nrequests);
tbin->tstats.nrequests = 0;
}
}
malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx);
if (lock_large) {
malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx);
}
unsigned ndeferred = 0;
for (unsigned i = 0; i < nflush; i++) {
......@@ -244,7 +298,7 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind,
extent = item_extent[i];
assert(ptr != NULL && extent != NULL);
if (extent_arena_get(extent) == locked_arena) {
if (extent_arena_ind_get(extent) == locked_arena_ind) {
large_dalloc_finish(tsd_tsdn(tsd), extent);
} else {
/*
......@@ -270,8 +324,8 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind,
* The flush loop didn't happen to flush to this thread's
* arena, so the stats didn't get merged. Manually do so now.
*/
arena_stats_large_nrequests_add(tsd_tsdn(tsd), &arena->stats,
binind, tbin->tstats.nrequests);
arena_stats_large_flush_nrequests_add(tsd_tsdn(tsd),
&tcache_arena->stats, binind, tbin->tstats.nrequests);
tbin->tstats.nrequests = 0;
}
......@@ -363,10 +417,10 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) {
size_t stack_offset = 0;
assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
memset(tcache->bins_small, 0, sizeof(cache_bin_t) * NBINS);
memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - NBINS));
memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS);
memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS));
unsigned i = 0;
for (; i < NBINS; i++) {
for (; i < SC_NBINS; i++) {
tcache->lg_fill_div[i] = 1;
stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
/*
......@@ -458,7 +512,7 @@ static void
tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
assert(tcache->arena != NULL);
for (unsigned i = 0; i < NBINS; i++) {
for (unsigned i = 0; i < SC_NBINS; i++) {
cache_bin_t *tbin = tcache_small_bin_get(tcache, i);
tcache_bin_flush_small(tsd, tcache, tbin, i, 0);
......@@ -466,7 +520,7 @@ tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
assert(tbin->tstats.nrequests == 0);
}
}
for (unsigned i = NBINS; i < nhbins; i++) {
for (unsigned i = SC_NBINS; i < nhbins; i++) {
cache_bin_t *tbin = tcache_large_bin_get(tcache, i);
tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
......@@ -491,6 +545,7 @@ tcache_flush(tsd_t *tsd) {
static void
tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
tcache_flush_cache(tsd, tcache);
arena_t *arena = tcache->arena;
tcache_arena_dissociate(tsd_tsdn(tsd), tcache);
if (tsd_tcache) {
......@@ -503,6 +558,23 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
/* Release both the tcache struct and avail array. */
idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true);
}
/*
* The deallocation and tcache flush above may not trigger decay since
* we are on the tcache shutdown path (potentially with non-nominal
* tsd). Manually trigger decay to avoid pathological cases. Also
* include arena 0 because the tcache array is allocated from it.
*/
arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false),
false, false);
if (arena_nthreads_get(arena, false) == 0 &&
!background_thread_enabled()) {
/* Force purging when no threads assigned to the arena anymore. */
arena_decay(tsd_tsdn(tsd), arena, false, true);
} else {
arena_decay(tsd_tsdn(tsd), arena, false, false);
}
}
/* For auto tcache (embedded in TSD) only. */
......@@ -532,10 +604,10 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
cassert(config_stats);
/* Merge and reset tcache stats. */
for (i = 0; i < NBINS; i++) {
bin_t *bin = &arena->bins[i];
for (i = 0; i < SC_NBINS; i++) {
cache_bin_t *tbin = tcache_small_bin_get(tcache, i);
malloc_mutex_lock(tsdn, &bin->lock);
unsigned binshard;
bin_t *bin = arena_bin_choose_lock(tsdn, arena, i, &binshard);
bin->stats.nrequests += tbin->tstats.nrequests;
malloc_mutex_unlock(tsdn, &bin->lock);
tbin->tstats.nrequests = 0;
......@@ -543,7 +615,7 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
for (; i < nhbins; i++) {
cache_bin_t *tbin = tcache_large_bin_get(tcache, i);
arena_stats_large_nrequests_add(tsdn, &arena->stats, i,
arena_stats_large_flush_nrequests_add(tsdn, &arena->stats, i,
tbin->tstats.nrequests);
tbin->tstats.nrequests = 0;
}
......@@ -614,23 +686,32 @@ label_return:
}
static tcache_t *
tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm) {
tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm, bool allow_reinit) {
malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
if (elm->tcache == NULL) {
return NULL;
}
tcache_t *tcache = elm->tcache;
elm->tcache = NULL;
if (allow_reinit) {
elm->tcache = TCACHES_ELM_NEED_REINIT;
} else {
elm->tcache = NULL;
}
if (tcache == TCACHES_ELM_NEED_REINIT) {
return NULL;
}
return tcache;
}
void
tcaches_flush(tsd_t *tsd, unsigned ind) {
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]);
tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true);
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
if (tcache != NULL) {
/* Destroy the tcache; recreate in tcaches_get() if needed. */
tcache_destroy(tsd, tcache, false);
}
}
......@@ -639,7 +720,7 @@ void
tcaches_destroy(tsd_t *tsd, unsigned ind) {
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
tcaches_t *elm = &tcaches[ind];
tcache_t *tcache = tcaches_elm_remove(tsd, elm);
tcache_t *tcache = tcaches_elm_remove(tsd, elm, false);
elm->next = tcaches_avail;
tcaches_avail = elm;
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
......@@ -652,8 +733,8 @@ bool
tcache_boot(tsdn_t *tsdn) {
/* If necessary, clamp opt_lg_tcache_max. */
if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) <
SMALL_MAXCLASS) {
tcache_maxclass = SMALL_MAXCLASS;
SC_SMALL_MAXCLASS) {
tcache_maxclass = SC_SMALL_MAXCLASS;
} else {
tcache_maxclass = (ZU(1) << opt_lg_tcache_max);
}
......@@ -673,7 +754,7 @@ tcache_boot(tsdn_t *tsdn) {
}
stack_nelms = 0;
unsigned i;
for (i = 0; i < NBINS; i++) {
for (i = 0; i < SC_NBINS; i++) {
if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) {
tcache_bin_info[i].ncached_max =
TCACHE_NSLOTS_SMALL_MIN;
......
......@@ -6,7 +6,7 @@
* from outside the generated library, so that we can use them in test code.
*/
JEMALLOC_EXPORT
void (*hooks_arena_new_hook)() = NULL;
void (*test_hooks_arena_new_hook)() = NULL;
JEMALLOC_EXPORT
void (*hooks_libc_hook)() = NULL;
void (*test_hooks_libc_hook)() = NULL;
......@@ -12,12 +12,16 @@
static unsigned ncleanups;
static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX];
/* TSD_INITIALIZER triggers "-Wmissing-field-initializer" */
JEMALLOC_DIAGNOSTIC_PUSH
JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
__thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER;
__thread bool JEMALLOC_TLS_MODEL tsd_initialized = false;
JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER;
JEMALLOC_TSD_TYPE_ATTR(bool) JEMALLOC_TLS_MODEL tsd_initialized = false;
bool tsd_booted = false;
#elif (defined(JEMALLOC_TLS))
__thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER;
JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER;
pthread_key_t tsd_tsd;
bool tsd_booted = false;
#elif (defined(_WIN32))
......@@ -41,6 +45,7 @@ tsd_init_head_t tsd_init_head = {
ql_head_initializer(blocks),
MALLOC_MUTEX_INITIALIZER
};
tsd_wrapper_t tsd_boot_wrapper = {
false,
TSD_INITIALIZER
......@@ -48,17 +53,164 @@ tsd_wrapper_t tsd_boot_wrapper = {
bool tsd_booted = false;
#endif
JEMALLOC_DIAGNOSTIC_POP
/******************************************************************************/
/* A list of all the tsds in the nominal state. */
typedef ql_head(tsd_t) tsd_list_t;
static tsd_list_t tsd_nominal_tsds = ql_head_initializer(tsd_nominal_tsds);
static malloc_mutex_t tsd_nominal_tsds_lock;
/* How many slow-path-enabling features are turned on. */
static atomic_u32_t tsd_global_slow_count = ATOMIC_INIT(0);
static bool
tsd_in_nominal_list(tsd_t *tsd) {
tsd_t *tsd_list;
bool found = false;
/*
* We don't know that tsd is nominal; it might not be safe to get data
* out of it here.
*/
malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock);
ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) {
if (tsd == tsd_list) {
found = true;
break;
}
}
malloc_mutex_unlock(TSDN_NULL, &tsd_nominal_tsds_lock);
return found;
}
static void
tsd_add_nominal(tsd_t *tsd) {
assert(!tsd_in_nominal_list(tsd));
assert(tsd_state_get(tsd) <= tsd_state_nominal_max);
ql_elm_new(tsd, TSD_MANGLE(tcache).tsd_link);
malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link);
malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
}
static void
tsd_remove_nominal(tsd_t *tsd) {
assert(tsd_in_nominal_list(tsd));
assert(tsd_state_get(tsd) <= tsd_state_nominal_max);
malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link);
malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
}
static void
tsd_force_recompute(tsdn_t *tsdn) {
/*
* The stores to tsd->state here need to synchronize with the exchange
* in tsd_slow_update.
*/
atomic_fence(ATOMIC_RELEASE);
malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock);
tsd_t *remote_tsd;
ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) {
assert(tsd_atomic_load(&remote_tsd->state, ATOMIC_RELAXED)
<= tsd_state_nominal_max);
tsd_atomic_store(&remote_tsd->state, tsd_state_nominal_recompute,
ATOMIC_RELAXED);
}
malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock);
}
void
tsd_global_slow_inc(tsdn_t *tsdn) {
atomic_fetch_add_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED);
/*
* We unconditionally force a recompute, even if the global slow count
* was already positive. If we didn't, then it would be possible for us
* to return to the user, have the user synchronize externally with some
* other thread, and then have that other thread not have picked up the
* update yet (since the original incrementing thread might still be
* making its way through the tsd list).
*/
tsd_force_recompute(tsdn);
}
void tsd_global_slow_dec(tsdn_t *tsdn) {
atomic_fetch_sub_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED);
/* See the note in ..._inc(). */
tsd_force_recompute(tsdn);
}
static bool
tsd_local_slow(tsd_t *tsd) {
return !tsd_tcache_enabled_get(tsd)
|| tsd_reentrancy_level_get(tsd) > 0;
}
bool
tsd_global_slow() {
return atomic_load_u32(&tsd_global_slow_count, ATOMIC_RELAXED) > 0;
}
/******************************************************************************/
static uint8_t
tsd_state_compute(tsd_t *tsd) {
if (!tsd_nominal(tsd)) {
return tsd_state_get(tsd);
}
/* We're in *a* nominal state; but which one? */
if (malloc_slow || tsd_local_slow(tsd) || tsd_global_slow()) {
return tsd_state_nominal_slow;
} else {
return tsd_state_nominal;
}
}
void
tsd_slow_update(tsd_t *tsd) {
if (tsd_nominal(tsd)) {
if (malloc_slow || !tsd_tcache_enabled_get(tsd) ||
tsd_reentrancy_level_get(tsd) > 0) {
tsd->state = tsd_state_nominal_slow;
uint8_t old_state;
do {
uint8_t new_state = tsd_state_compute(tsd);
old_state = tsd_atomic_exchange(&tsd->state, new_state,
ATOMIC_ACQUIRE);
} while (old_state == tsd_state_nominal_recompute);
}
void
tsd_state_set(tsd_t *tsd, uint8_t new_state) {
/* Only the tsd module can change the state *to* recompute. */
assert(new_state != tsd_state_nominal_recompute);
uint8_t old_state = tsd_atomic_load(&tsd->state, ATOMIC_RELAXED);
if (old_state > tsd_state_nominal_max) {
/*
* Not currently in the nominal list, but it might need to be
* inserted there.
*/
assert(!tsd_in_nominal_list(tsd));
tsd_atomic_store(&tsd->state, new_state, ATOMIC_RELAXED);
if (new_state <= tsd_state_nominal_max) {
tsd_add_nominal(tsd);
}
} else {
/*
* We're currently nominal. If the new state is non-nominal,
* great; we take ourselves off the list and just enter the new
* state.
*/
assert(tsd_in_nominal_list(tsd));
if (new_state > tsd_state_nominal_max) {
tsd_remove_nominal(tsd);
tsd_atomic_store(&tsd->state, new_state,
ATOMIC_RELAXED);
} else {
tsd->state = tsd_state_nominal;
/*
* This is the tricky case. We're transitioning from
* one nominal state to another. The caller can't know
* about any races that are occuring at the same time,
* so we always have to recompute no matter what.
*/
tsd_slow_update(tsd);
}
}
}
......@@ -87,6 +239,7 @@ tsd_data_init(tsd_t *tsd) {
static void
assert_tsd_data_cleanup_done(tsd_t *tsd) {
assert(!tsd_nominal(tsd));
assert(!tsd_in_nominal_list(tsd));
assert(*tsd_arenap_get_unsafe(tsd) == NULL);
assert(*tsd_iarenap_get_unsafe(tsd) == NULL);
assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true);
......@@ -97,8 +250,8 @@ assert_tsd_data_cleanup_done(tsd_t *tsd) {
static bool
tsd_data_init_nocleanup(tsd_t *tsd) {
assert(tsd->state == tsd_state_reincarnated ||
tsd->state == tsd_state_minimal_initialized);
assert(tsd_state_get(tsd) == tsd_state_reincarnated ||
tsd_state_get(tsd) == tsd_state_minimal_initialized);
/*
* During reincarnation, there is no guarantee that the cleanup function
* will be called (deallocation may happen after all tsd destructors).
......@@ -117,27 +270,33 @@ tsd_t *
tsd_fetch_slow(tsd_t *tsd, bool minimal) {
assert(!tsd_fast(tsd));
if (tsd->state == tsd_state_nominal_slow) {
/* On slow path but no work needed. */
assert(malloc_slow || !tsd_tcache_enabled_get(tsd) ||
tsd_reentrancy_level_get(tsd) > 0 ||
*tsd_arenas_tdata_bypassp_get(tsd));
} else if (tsd->state == tsd_state_uninitialized) {
if (tsd_state_get(tsd) == tsd_state_nominal_slow) {
/*
* On slow path but no work needed. Note that we can't
* necessarily *assert* that we're slow, because we might be
* slow because of an asynchronous modification to global state,
* which might be asynchronously modified *back*.
*/
} else if (tsd_state_get(tsd) == tsd_state_nominal_recompute) {
tsd_slow_update(tsd);
} else if (tsd_state_get(tsd) == tsd_state_uninitialized) {
if (!minimal) {
tsd->state = tsd_state_nominal;
tsd_slow_update(tsd);
/* Trigger cleanup handler registration. */
tsd_set(tsd);
tsd_data_init(tsd);
if (tsd_booted) {
tsd_state_set(tsd, tsd_state_nominal);
tsd_slow_update(tsd);
/* Trigger cleanup handler registration. */
tsd_set(tsd);
tsd_data_init(tsd);
}
} else {
tsd->state = tsd_state_minimal_initialized;
tsd_state_set(tsd, tsd_state_minimal_initialized);
tsd_set(tsd);
tsd_data_init_nocleanup(tsd);
}
} else if (tsd->state == tsd_state_minimal_initialized) {
} else if (tsd_state_get(tsd) == tsd_state_minimal_initialized) {
if (!minimal) {
/* Switch to fully initialized. */
tsd->state = tsd_state_nominal;
tsd_state_set(tsd, tsd_state_nominal);
assert(*tsd_reentrancy_levelp_get(tsd) >= 1);
(*tsd_reentrancy_levelp_get(tsd))--;
tsd_slow_update(tsd);
......@@ -145,12 +304,12 @@ tsd_fetch_slow(tsd_t *tsd, bool minimal) {
} else {
assert_tsd_data_cleanup_done(tsd);
}
} else if (tsd->state == tsd_state_purgatory) {
tsd->state = tsd_state_reincarnated;
} else if (tsd_state_get(tsd) == tsd_state_purgatory) {
tsd_state_set(tsd, tsd_state_reincarnated);
tsd_set(tsd);
tsd_data_init_nocleanup(tsd);
} else {
assert(tsd->state == tsd_state_reincarnated);
assert(tsd_state_get(tsd) == tsd_state_reincarnated);
}
return tsd;
......@@ -214,7 +373,7 @@ void
tsd_cleanup(void *arg) {
tsd_t *tsd = (tsd_t *)arg;
switch (tsd->state) {
switch (tsd_state_get(tsd)) {
case tsd_state_uninitialized:
/* Do nothing. */
break;
......@@ -232,7 +391,7 @@ tsd_cleanup(void *arg) {
case tsd_state_nominal:
case tsd_state_nominal_slow:
tsd_do_data_cleanup(tsd);
tsd->state = tsd_state_purgatory;
tsd_state_set(tsd, tsd_state_purgatory);
tsd_set(tsd);
break;
case tsd_state_purgatory:
......@@ -260,6 +419,10 @@ malloc_tsd_boot0(void) {
tsd_t *tsd;
ncleanups = 0;
if (malloc_mutex_init(&tsd_nominal_tsds_lock, "tsd_nominal_tsds_lock",
WITNESS_RANK_OMIT, malloc_mutex_rank_exclusive)) {
return NULL;
}
if (tsd_boot0()) {
return NULL;
}
......@@ -310,7 +473,7 @@ _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
# pragma comment(linker, "/INCLUDE:_tls_callback")
# else
# pragma comment(linker, "/INCLUDE:_tls_used")
# pragma comment(linker, "/INCLUDE:tls_callback")
# pragma comment(linker, "/INCLUDE:" STRINGIFY(tls_callback) )
# endif
# pragma section(".CRT$XLY",long,read)
#endif
......@@ -349,3 +512,23 @@ tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) {
malloc_mutex_unlock(TSDN_NULL, &head->lock);
}
#endif
void
tsd_prefork(tsd_t *tsd) {
malloc_mutex_prefork(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
}
void
tsd_postfork_parent(tsd_t *tsd) {
malloc_mutex_postfork_parent(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
}
void
tsd_postfork_child(tsd_t *tsd) {
malloc_mutex_postfork_child(tsd_tsdn(tsd), &tsd_nominal_tsds_lock);
ql_new(&tsd_nominal_tsds);
if (tsd_state_get(tsd) <= tsd_state_nominal_max) {
tsd_add_nominal(tsd);
}
}
......@@ -25,7 +25,7 @@ extern "C" {
#include "test/jemalloc_test_defs.h"
#ifdef JEMALLOC_OSSPIN
#if defined(JEMALLOC_OSATOMIC)
# include <libkern/OSAtomic.h>
#endif
......@@ -69,7 +69,7 @@ static const bool config_debug =
# define JEMALLOC_N(n) @private_namespace@##n
# include "jemalloc/internal/private_namespace.h"
# include "jemalloc/internal/hooks.h"
# include "jemalloc/internal/test_hooks.h"
/* Hermetic headers. */
# include "jemalloc/internal/assert.h"
......
......@@ -10,8 +10,6 @@ typedef struct {
CRITICAL_SECTION lock;
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
os_unfair_lock lock;
#elif (defined(JEMALLOC_OSSPIN))
OSSpinLock lock;
#else
pthread_mutex_t lock;
#endif
......
......@@ -34,6 +34,17 @@ TEST_BEGIN(test_alignment_errors) {
}
TEST_END
/*
* GCC "-Walloc-size-larger-than" warning detects when one of the memory
* allocation functions is called with a size larger than the maximum size that
* they support. Here we want to explicitly test that the allocation functions
* do indeed fail properly when this is the case, which triggers the warning.
* Therefore we disable the warning for these tests.
*/
JEMALLOC_DIAGNOSTIC_PUSH
JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN
TEST_BEGIN(test_oom_errors) {
size_t alignment, size;
void *p;
......@@ -78,6 +89,9 @@ TEST_BEGIN(test_oom_errors) {
}
TEST_END
/* Re-enable the "-Walloc-size-larger-than=" warning */
JEMALLOC_DIAGNOSTIC_POP
TEST_BEGIN(test_alignment_and_size) {
#define NITER 4
size_t alignment, size, total;
......@@ -124,10 +138,20 @@ TEST_BEGIN(test_alignment_and_size) {
}
TEST_END
TEST_BEGIN(test_zero_alloc) {
void *res = aligned_alloc(8, 0);
assert(res);
size_t usable = malloc_usable_size(res);
assert(usable > 0);
free(res);
}
TEST_END
int
main(void) {
return test(
test_alignment_errors,
test_oom_errors,
test_alignment_and_size);
test_alignment_and_size,
test_zero_alloc);
}
#include "test/jemalloc_test.h"
TEST_BEGIN(test_zero_alloc) {
void *res = malloc(0);
assert(res);
size_t usable = malloc_usable_size(res);
assert(usable > 0);
free(res);
}
TEST_END
int
main(void) {
return test(
test_zero_alloc);
}
......@@ -51,6 +51,16 @@ purge(void) {
"Unexpected mallctl error");
}
/*
* GCC "-Walloc-size-larger-than" warning detects when one of the memory
* allocation functions is called with a size larger than the maximum size that
* they support. Here we want to explicitly test that the allocation functions
* do indeed fail properly when this is the case, which triggers the warning.
* Therefore we disable the warning for these tests.
*/
JEMALLOC_DIAGNOSTIC_PUSH
JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN
TEST_BEGIN(test_overflow) {
size_t largemax;
......@@ -71,6 +81,38 @@ TEST_BEGIN(test_overflow) {
}
TEST_END
static void *
remote_alloc(void *arg) {
unsigned arena;
size_t sz = sizeof(unsigned);
assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0,
"Unexpected mallctl() failure");
size_t large_sz;
sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large_sz, &sz,
NULL, 0), 0, "Unexpected mallctl failure");
void *ptr = mallocx(large_sz, MALLOCX_ARENA(arena)
| MALLOCX_TCACHE_NONE);
void **ret = (void **)arg;
*ret = ptr;
return NULL;
}
TEST_BEGIN(test_remote_free) {
thd_t thd;
void *ret;
thd_create(&thd, remote_alloc, (void *)&ret);
thd_join(thd, NULL);
assert_ptr_not_null(ret, "Unexpected mallocx failure");
/* Avoid TCACHE_NONE to explicitly test tcache_flush(). */
dallocx(ret, 0);
mallctl("thread.tcache.flush", NULL, NULL, NULL, 0);
}
TEST_END
TEST_BEGIN(test_oom) {
size_t largemax;
bool oom;
......@@ -84,7 +126,7 @@ TEST_BEGIN(test_oom) {
largemax = get_large_size(get_nlarge()-1);
oom = false;
for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) {
ptrs[i] = mallocx(largemax, 0);
ptrs[i] = mallocx(largemax, MALLOCX_ARENA(0));
if (ptrs[i] == NULL) {
oom = true;
}
......@@ -113,6 +155,9 @@ TEST_BEGIN(test_oom) {
}
TEST_END
/* Re-enable the "-Walloc-size-larger-than=" warning */
JEMALLOC_DIAGNOSTIC_POP
TEST_BEGIN(test_basic) {
#define MAXSZ (((size_t)1) << 23)
size_t sz;
......@@ -178,12 +223,12 @@ TEST_BEGIN(test_alignment_and_size) {
sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) {
for (i = 0; i < NITER; i++) {
nsz = nallocx(sz, MALLOCX_ALIGN(alignment) |
MALLOCX_ZERO);
MALLOCX_ZERO | MALLOCX_ARENA(0));
assert_zu_ne(nsz, 0,
"nallocx() error for alignment=%zu, "
"size=%zu (%#zx)", alignment, sz, sz);
ps[i] = mallocx(sz, MALLOCX_ALIGN(alignment) |
MALLOCX_ZERO);
MALLOCX_ZERO | MALLOCX_ARENA(0));
assert_ptr_not_null(ps[i],
"mallocx() error for alignment=%zu, "
"size=%zu (%#zx)", alignment, sz, sz);
......@@ -223,6 +268,7 @@ main(void) {
return test(
test_overflow,
test_oom,
test_remote_free,
test_basic,
test_alignment_and_size);
}
#include "test/jemalloc_test.h"
/*
* GCC "-Walloc-size-larger-than" warning detects when one of the memory
* allocation functions is called with a size larger than the maximum size that
* they support. Here we want to explicitly test that the allocation functions
* do indeed fail properly when this is the case, which triggers the warning.
* Therefore we disable the warning for these tests.
*/
JEMALLOC_DIAGNOSTIC_PUSH
JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN
TEST_BEGIN(test_overflow) {
unsigned nlextents;
size_t mib[4];
......@@ -39,6 +49,9 @@ TEST_BEGIN(test_overflow) {
}
TEST_END
/* Re-enable the "-Walloc-size-larger-than=" warning */
JEMALLOC_DIAGNOSTIC_POP
int
main(void) {
return test(
......
......@@ -85,9 +85,10 @@ TEST_BEGIN(test_alignment_and_size) {
alignment <= MAXALIGN;
alignment <<= 1) {
total = 0;
for (size = 1;
for (size = 0;
size < 3 * alignment && size < (1U << 31);
size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) {
size += ((size == 0) ? 1 :
(alignment >> (LG_SIZEOF_PTR-1)) - 1)) {
for (i = 0; i < NITER; i++) {
err = posix_memalign(&ps[i],
alignment, size);
......
......@@ -208,6 +208,16 @@ TEST_BEGIN(test_lg_align_and_zero) {
}
TEST_END
/*
* GCC "-Walloc-size-larger-than" warning detects when one of the memory
* allocation functions is called with a size larger than the maximum size that
* they support. Here we want to explicitly test that the allocation functions
* do indeed fail properly when this is the case, which triggers the warning.
* Therefore we disable the warning for these tests.
*/
JEMALLOC_DIAGNOSTIC_PUSH
JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN
TEST_BEGIN(test_overflow) {
size_t largemax;
void *p;
......@@ -234,6 +244,9 @@ TEST_BEGIN(test_overflow) {
}
TEST_END
/* Re-enable the "-Walloc-size-larger-than=" warning */
JEMALLOC_DIAGNOSTIC_POP
int
main(void) {
return test(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment