Unverified Commit a51eb05b authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Release Redis 7.2 RC2

parents e26a769d 986dbf71
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/fxp.h"
static bool
fxp_isdigit(char c) {
return '0' <= c && c <= '9';
}
bool
fxp_parse(fxp_t *result, const char *str, char **end) {
/*
* Using malloc_strtoumax in this method isn't as handy as you might
* expect (I tried). In the fractional part, significant leading zeros
* mean that you still need to do your own parsing, now with trickier
* math. In the integer part, the casting (uintmax_t to uint32_t)
* forces more reasoning about bounds than just checking for overflow as
* we parse.
*/
uint32_t integer_part = 0;
const char *cur = str;
/* The string must start with a digit or a decimal point. */
if (*cur != '.' && !fxp_isdigit(*cur)) {
return true;
}
while ('0' <= *cur && *cur <= '9') {
integer_part *= 10;
integer_part += *cur - '0';
if (integer_part >= (1U << 16)) {
return true;
}
cur++;
}
/*
* We've parsed all digits at the beginning of the string, without
* overflow. Either we're done, or there's a fractional part.
*/
if (*cur != '.') {
*result = (integer_part << 16);
if (end != NULL) {
*end = (char *)cur;
}
return false;
}
/* There's a fractional part. */
cur++;
if (!fxp_isdigit(*cur)) {
/* Shouldn't end on the decimal point. */
return true;
}
/*
* We use a lot of precision for the fractional part, even though we'll
* discard most of it; this lets us get exact values for the important
* special case where the denominator is a small power of 2 (for
* instance, 1/512 == 0.001953125 is exactly representable even with
* only 16 bits of fractional precision). We need to left-shift by 16
* before dividing so we pick the number of digits to be
* floor(log(2**48)) = 14.
*/
uint64_t fractional_part = 0;
uint64_t frac_div = 1;
for (int i = 0; i < FXP_FRACTIONAL_PART_DIGITS; i++) {
fractional_part *= 10;
frac_div *= 10;
if (fxp_isdigit(*cur)) {
fractional_part += *cur - '0';
cur++;
}
}
/*
* We only parse the first maxdigits characters, but we can still ignore
* any digits after that.
*/
while (fxp_isdigit(*cur)) {
cur++;
}
assert(fractional_part < frac_div);
uint32_t fractional_repr = (uint32_t)(
(fractional_part << 16) / frac_div);
/* Success! */
*result = (integer_part << 16) + fractional_repr;
if (end != NULL) {
*end = (char *)cur;
}
return false;
}
void
fxp_print(fxp_t a, char buf[FXP_BUF_SIZE]) {
uint32_t integer_part = fxp_round_down(a);
uint32_t fractional_part = (a & ((1U << 16) - 1));
int leading_fraction_zeros = 0;
uint64_t fraction_digits = fractional_part;
for (int i = 0; i < FXP_FRACTIONAL_PART_DIGITS; i++) {
if (fraction_digits < (1U << 16)
&& fraction_digits * 10 >= (1U << 16)) {
leading_fraction_zeros = i;
}
fraction_digits *= 10;
}
fraction_digits >>= 16;
while (fraction_digits > 0 && fraction_digits % 10 == 0) {
fraction_digits /= 10;
}
size_t printed = malloc_snprintf(buf, FXP_BUF_SIZE, "%"FMTu32".",
integer_part);
for (int i = 0; i < leading_fraction_zeros; i++) {
buf[printed] = '0';
printed++;
}
malloc_snprintf(&buf[printed], FXP_BUF_SIZE - printed, "%"FMTu64,
fraction_digits);
}
......@@ -130,9 +130,9 @@ hook_reentrantp() {
*/
static bool in_hook_global = true;
tsdn_t *tsdn = tsdn_fetch();
tcache_t *tcache = tsdn_tcachep_get(tsdn);
if (tcache != NULL) {
return &tcache->in_hook;
bool *in_hook = tsdn_in_hookp_get(tsdn);
if (in_hook!= NULL) {
return in_hook;
}
return &in_hook_global;
}
......
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/hpa.h"
#include "jemalloc/internal/fb.h"
#include "jemalloc/internal/witness.h"
#define HPA_EDEN_SIZE (128 * HUGEPAGE)
static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
static size_t hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size,
size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated);
static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool *deferred_work_generated);
static void hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated);
static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
edata_list_active_t *list, bool *deferred_work_generated);
static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
bool
hpa_supported() {
#ifdef _WIN32
/*
* At least until the API and implementation is somewhat settled, we
* don't want to try to debug the VM subsystem on the hardest-to-test
* platform.
*/
return false;
#endif
if (!pages_can_hugify) {
return false;
}
/*
* We fundamentally rely on a address-space-hungry growth strategy for
* hugepages.
*/
if (LG_SIZEOF_PTR != 3) {
return false;
}
/*
* If we couldn't detect the value of HUGEPAGE, HUGEPAGE_PAGES becomes
* this sentinel value -- see the comment in pages.h.
*/
if (HUGEPAGE_PAGES == 1) {
return false;
}
return true;
}
static void
hpa_do_consistency_checks(hpa_shard_t *shard) {
assert(shard->base != NULL);
}
bool
hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) {
/* malloc_conf processing should have filtered out these cases. */
assert(hpa_supported());
bool err;
err = malloc_mutex_init(&central->grow_mtx, "hpa_central_grow",
WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive);
if (err) {
return true;
}
err = malloc_mutex_init(&central->mtx, "hpa_central",
WITNESS_RANK_HPA_CENTRAL, malloc_mutex_rank_exclusive);
if (err) {
return true;
}
central->base = base;
central->eden = NULL;
central->eden_len = 0;
central->age_counter = 0;
central->hooks = *hooks;
return false;
}
static hpdata_t *
hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) {
return (hpdata_t *)base_alloc(tsdn, central->base, sizeof(hpdata_t),
CACHELINE);
}
hpdata_t *
hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
bool *oom) {
/* Don't yet support big allocations; these should get filtered out. */
assert(size <= HUGEPAGE);
/*
* Should only try to extract from the central allocator if the local
* shard is exhausted. We should hold the grow_mtx on that shard.
*/
witness_assert_positive_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW);
malloc_mutex_lock(tsdn, &central->grow_mtx);
*oom = false;
hpdata_t *ps = NULL;
/* Is eden a perfect fit? */
if (central->eden != NULL && central->eden_len == HUGEPAGE) {
ps = hpa_alloc_ps(tsdn, central);
if (ps == NULL) {
*oom = true;
malloc_mutex_unlock(tsdn, &central->grow_mtx);
return NULL;
}
hpdata_init(ps, central->eden, central->age_counter++);
central->eden = NULL;
central->eden_len = 0;
malloc_mutex_unlock(tsdn, &central->grow_mtx);
return ps;
}
/*
* We're about to try to allocate from eden by splitting. If eden is
* NULL, we have to allocate it too. Otherwise, we just have to
* allocate an edata_t for the new psset.
*/
if (central->eden == NULL) {
/*
* During development, we're primarily concerned with systems
* with overcommit. Eventually, we should be more careful here.
*/
bool commit = true;
/* Allocate address space, bailing if we fail. */
void *new_eden = pages_map(NULL, HPA_EDEN_SIZE, HUGEPAGE,
&commit);
if (new_eden == NULL) {
*oom = true;
malloc_mutex_unlock(tsdn, &central->grow_mtx);
return NULL;
}
ps = hpa_alloc_ps(tsdn, central);
if (ps == NULL) {
pages_unmap(new_eden, HPA_EDEN_SIZE);
*oom = true;
malloc_mutex_unlock(tsdn, &central->grow_mtx);
return NULL;
}
central->eden = new_eden;
central->eden_len = HPA_EDEN_SIZE;
} else {
/* Eden is already nonempty; only need an edata for ps. */
ps = hpa_alloc_ps(tsdn, central);
if (ps == NULL) {
*oom = true;
malloc_mutex_unlock(tsdn, &central->grow_mtx);
return NULL;
}
}
assert(ps != NULL);
assert(central->eden != NULL);
assert(central->eden_len > HUGEPAGE);
assert(central->eden_len % HUGEPAGE == 0);
assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden);
hpdata_init(ps, central->eden, central->age_counter++);
char *eden_char = (char *)central->eden;
eden_char += HUGEPAGE;
central->eden = (void *)eden_char;
central->eden_len -= HUGEPAGE;
malloc_mutex_unlock(tsdn, &central->grow_mtx);
return ps;
}
bool
hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
base_t *base, edata_cache_t *edata_cache, unsigned ind,
const hpa_shard_opts_t *opts) {
/* malloc_conf processing should have filtered out these cases. */
assert(hpa_supported());
bool err;
err = malloc_mutex_init(&shard->grow_mtx, "hpa_shard_grow",
WITNESS_RANK_HPA_SHARD_GROW, malloc_mutex_rank_exclusive);
if (err) {
return true;
}
err = malloc_mutex_init(&shard->mtx, "hpa_shard",
WITNESS_RANK_HPA_SHARD, malloc_mutex_rank_exclusive);
if (err) {
return true;
}
assert(edata_cache != NULL);
shard->central = central;
shard->base = base;
edata_cache_fast_init(&shard->ecf, edata_cache);
psset_init(&shard->psset);
shard->age_counter = 0;
shard->ind = ind;
shard->emap = emap;
shard->opts = *opts;
shard->npending_purge = 0;
nstime_init_zero(&shard->last_purge);
shard->stats.npurge_passes = 0;
shard->stats.npurges = 0;
shard->stats.nhugifies = 0;
shard->stats.ndehugifies = 0;
/*
* Fill these in last, so that if an hpa_shard gets used despite
* initialization failing, we'll at least crash instead of just
* operating on corrupted data.
*/
shard->pai.alloc = &hpa_alloc;
shard->pai.alloc_batch = &hpa_alloc_batch;
shard->pai.expand = &hpa_expand;
shard->pai.shrink = &hpa_shrink;
shard->pai.dalloc = &hpa_dalloc;
shard->pai.dalloc_batch = &hpa_dalloc_batch;
shard->pai.time_until_deferred_work = &hpa_time_until_deferred_work;
hpa_do_consistency_checks(shard);
return false;
}
/*
* Note that the stats functions here follow the usual stats naming conventions;
* "merge" obtains the stats from some live object of instance, while "accum"
* only combines the stats from one stats objet to another. Hence the lack of
* locking here.
*/
static void
hpa_shard_nonderived_stats_accum(hpa_shard_nonderived_stats_t *dst,
hpa_shard_nonderived_stats_t *src) {
dst->npurge_passes += src->npurge_passes;
dst->npurges += src->npurges;
dst->nhugifies += src->nhugifies;
dst->ndehugifies += src->ndehugifies;
}
void
hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src) {
psset_stats_accum(&dst->psset_stats, &src->psset_stats);
hpa_shard_nonderived_stats_accum(&dst->nonderived_stats,
&src->nonderived_stats);
}
void
hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard,
hpa_shard_stats_t *dst) {
hpa_do_consistency_checks(shard);
malloc_mutex_lock(tsdn, &shard->grow_mtx);
malloc_mutex_lock(tsdn, &shard->mtx);
psset_stats_accum(&dst->psset_stats, &shard->psset.stats);
hpa_shard_nonderived_stats_accum(&dst->nonderived_stats, &shard->stats);
malloc_mutex_unlock(tsdn, &shard->mtx);
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
}
static bool
hpa_good_hugification_candidate(hpa_shard_t *shard, hpdata_t *ps) {
/*
* Note that this needs to be >= rather than just >, because of the
* important special case in which the hugification threshold is exactly
* HUGEPAGE.
*/
return hpdata_nactive_get(ps) * PAGE
>= shard->opts.hugification_threshold;
}
static size_t
hpa_adjusted_ndirty(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
return psset_ndirty(&shard->psset) - shard->npending_purge;
}
static size_t
hpa_ndirty_max(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
if (shard->opts.dirty_mult == (fxp_t)-1) {
return (size_t)-1;
}
return fxp_mul_frac(psset_nactive(&shard->psset),
shard->opts.dirty_mult);
}
static bool
hpa_hugify_blocked_by_ndirty(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
hpdata_t *to_hugify = psset_pick_hugify(&shard->psset);
if (to_hugify == NULL) {
return false;
}
return hpa_adjusted_ndirty(tsdn, shard)
+ hpdata_nretained_get(to_hugify) > hpa_ndirty_max(tsdn, shard);
}
static bool
hpa_should_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
if (hpa_adjusted_ndirty(tsdn, shard) > hpa_ndirty_max(tsdn, shard)) {
return true;
}
if (hpa_hugify_blocked_by_ndirty(tsdn, shard)) {
return true;
}
return false;
}
static void
hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard,
hpdata_t *ps) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
if (hpdata_changing_state_get(ps)) {
hpdata_purge_allowed_set(ps, false);
hpdata_disallow_hugify(ps);
return;
}
/*
* Hugepages are distinctly costly to purge, so try to avoid it unless
* they're *particularly* full of dirty pages. Eventually, we should
* use a smarter / more dynamic heuristic for situations where we have
* to manually hugify.
*
* In situations where we don't manually hugify, this problem is
* reduced. The "bad" situation we're trying to avoid is one's that's
* common in some Linux configurations (where both enabled and defrag
* are set to madvise) that can lead to long latency spikes on the first
* access after a hugification. The ideal policy in such configurations
* is probably time-based for both purging and hugifying; only hugify a
* hugepage if it's met the criteria for some extended period of time,
* and only dehugify it if it's failed to meet the criteria for an
* extended period of time. When background threads are on, we should
* try to take this hit on one of them, as well.
*
* I think the ideal setting is THP always enabled, and defrag set to
* deferred; in that case we don't need any explicit calls on the
* allocator's end at all; we just try to pack allocations in a
* hugepage-friendly manner and let the OS hugify in the background.
*/
hpdata_purge_allowed_set(ps, hpdata_ndirty_get(ps) > 0);
if (hpa_good_hugification_candidate(shard, ps)
&& !hpdata_huge_get(ps)) {
nstime_t now;
shard->central->hooks.curtime(&now, /* first_reading */ true);
hpdata_allow_hugify(ps, now);
}
/*
* Once a hugepage has become eligible for hugification, we don't mark
* it as ineligible just because it stops meeting the criteria (this
* could lead to situations where a hugepage that spends most of its
* time meeting the criteria never quite getting hugified if there are
* intervening deallocations). The idea is that the hugification delay
* will allow them to get purged, reseting their "hugify-allowed" bit.
* If they don't get purged, then the hugification isn't hurting and
* might help. As an exception, we don't hugify hugepages that are now
* empty; it definitely doesn't help there until the hugepage gets
* reused, which is likely not for a while.
*/
if (hpdata_nactive_get(ps) == 0) {
hpdata_disallow_hugify(ps);
}
}
static bool
hpa_shard_has_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
hpdata_t *to_hugify = psset_pick_hugify(&shard->psset);
return to_hugify != NULL || hpa_should_purge(tsdn, shard);
}
/* Returns whether or not we purged anything. */
static bool
hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
hpdata_t *to_purge = psset_pick_purge(&shard->psset);
if (to_purge == NULL) {
return false;
}
assert(hpdata_purge_allowed_get(to_purge));
assert(!hpdata_changing_state_get(to_purge));
/*
* Don't let anyone else purge or hugify this page while
* we're purging it (allocations and deallocations are
* OK).
*/
psset_update_begin(&shard->psset, to_purge);
assert(hpdata_alloc_allowed_get(to_purge));
hpdata_mid_purge_set(to_purge, true);
hpdata_purge_allowed_set(to_purge, false);
hpdata_disallow_hugify(to_purge);
/*
* Unlike with hugification (where concurrent
* allocations are allowed), concurrent allocation out
* of a hugepage being purged is unsafe; we might hand
* out an extent for an allocation and then purge it
* (clearing out user data).
*/
hpdata_alloc_allowed_set(to_purge, false);
psset_update_end(&shard->psset, to_purge);
/* Gather all the metadata we'll need during the purge. */
bool dehugify = hpdata_huge_get(to_purge);
hpdata_purge_state_t purge_state;
size_t num_to_purge = hpdata_purge_begin(to_purge, &purge_state);
shard->npending_purge += num_to_purge;
malloc_mutex_unlock(tsdn, &shard->mtx);
/* Actually do the purging, now that the lock is dropped. */
if (dehugify) {
shard->central->hooks.dehugify(hpdata_addr_get(to_purge),
HUGEPAGE);
}
size_t total_purged = 0;
uint64_t purges_this_pass = 0;
void *purge_addr;
size_t purge_size;
while (hpdata_purge_next(to_purge, &purge_state, &purge_addr,
&purge_size)) {
total_purged += purge_size;
assert(total_purged <= HUGEPAGE);
purges_this_pass++;
shard->central->hooks.purge(purge_addr, purge_size);
}
malloc_mutex_lock(tsdn, &shard->mtx);
/* The shard updates */
shard->npending_purge -= num_to_purge;
shard->stats.npurge_passes++;
shard->stats.npurges += purges_this_pass;
shard->central->hooks.curtime(&shard->last_purge,
/* first_reading */ false);
if (dehugify) {
shard->stats.ndehugifies++;
}
/* The hpdata updates. */
psset_update_begin(&shard->psset, to_purge);
if (dehugify) {
hpdata_dehugify(to_purge);
}
hpdata_purge_end(to_purge, &purge_state);
hpdata_mid_purge_set(to_purge, false);
hpdata_alloc_allowed_set(to_purge, true);
hpa_update_purge_hugify_eligibility(tsdn, shard, to_purge);
psset_update_end(&shard->psset, to_purge);
return true;
}
/* Returns whether or not we hugified anything. */
static bool
hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
if (hpa_hugify_blocked_by_ndirty(tsdn, shard)) {
return false;
}
hpdata_t *to_hugify = psset_pick_hugify(&shard->psset);
if (to_hugify == NULL) {
return false;
}
assert(hpdata_hugify_allowed_get(to_hugify));
assert(!hpdata_changing_state_get(to_hugify));
/* Make sure that it's been hugifiable for long enough. */
nstime_t time_hugify_allowed = hpdata_time_hugify_allowed(to_hugify);
uint64_t millis = shard->central->hooks.ms_since(&time_hugify_allowed);
if (millis < shard->opts.hugify_delay_ms) {
return false;
}
/*
* Don't let anyone else purge or hugify this page while
* we're hugifying it (allocations and deallocations are
* OK).
*/
psset_update_begin(&shard->psset, to_hugify);
hpdata_mid_hugify_set(to_hugify, true);
hpdata_purge_allowed_set(to_hugify, false);
hpdata_disallow_hugify(to_hugify);
assert(hpdata_alloc_allowed_get(to_hugify));
psset_update_end(&shard->psset, to_hugify);
malloc_mutex_unlock(tsdn, &shard->mtx);
shard->central->hooks.hugify(hpdata_addr_get(to_hugify), HUGEPAGE);
malloc_mutex_lock(tsdn, &shard->mtx);
shard->stats.nhugifies++;
psset_update_begin(&shard->psset, to_hugify);
hpdata_hugify(to_hugify);
hpdata_mid_hugify_set(to_hugify, false);
hpa_update_purge_hugify_eligibility(tsdn, shard, to_hugify);
psset_update_end(&shard->psset, to_hugify);
return true;
}
/*
* Execution of deferred work is forced if it's triggered by an explicit
* hpa_shard_do_deferred_work() call.
*/
static void
hpa_shard_maybe_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard,
bool forced) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
if (!forced && shard->opts.deferral_allowed) {
return;
}
/*
* If we're on a background thread, do work so long as there's work to
* be done. Otherwise, bound latency to not be *too* bad by doing at
* most a small fixed number of operations.
*/
bool hugified = false;
bool purged = false;
size_t max_ops = (forced ? (size_t)-1 : 16);
size_t nops = 0;
do {
/*
* Always purge before hugifying, to make sure we get some
* ability to hit our quiescence targets.
*/
purged = false;
while (hpa_should_purge(tsdn, shard) && nops < max_ops) {
purged = hpa_try_purge(tsdn, shard);
if (purged) {
nops++;
}
}
hugified = hpa_try_hugify(tsdn, shard);
if (hugified) {
nops++;
}
malloc_mutex_assert_owner(tsdn, &shard->mtx);
malloc_mutex_assert_owner(tsdn, &shard->mtx);
} while ((hugified || purged) && nops < max_ops);
}
static edata_t *
hpa_try_alloc_one_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
bool *oom) {
bool err;
edata_t *edata = edata_cache_fast_get(tsdn, &shard->ecf);
if (edata == NULL) {
*oom = true;
return NULL;
}
hpdata_t *ps = psset_pick_alloc(&shard->psset, size);
if (ps == NULL) {
edata_cache_fast_put(tsdn, &shard->ecf, edata);
return NULL;
}
psset_update_begin(&shard->psset, ps);
if (hpdata_empty(ps)) {
/*
* If the pageslab used to be empty, treat it as though it's
* brand new for fragmentation-avoidance purposes; what we're
* trying to approximate is the age of the allocations *in* that
* pageslab, and the allocations in the new pageslab are
* definitionally the youngest in this hpa shard.
*/
hpdata_age_set(ps, shard->age_counter++);
}
void *addr = hpdata_reserve_alloc(ps, size);
edata_init(edata, shard->ind, addr, size, /* slab */ false,
SC_NSIZES, /* sn */ hpdata_age_get(ps), extent_state_active,
/* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
EXTENT_NOT_HEAD);
edata_ps_set(edata, ps);
/*
* This could theoretically be moved outside of the critical section,
* but that introduces the potential for a race. Without the lock, the
* (initially nonempty, since this is the reuse pathway) pageslab we
* allocated out of could become otherwise empty while the lock is
* dropped. This would force us to deal with a pageslab eviction down
* the error pathway, which is a pain.
*/
err = emap_register_boundary(tsdn, shard->emap, edata,
SC_NSIZES, /* slab */ false);
if (err) {
hpdata_unreserve(ps, edata_addr_get(edata),
edata_size_get(edata));
/*
* We should arguably reset dirty state here, but this would
* require some sort of prepare + commit functionality that's a
* little much to deal with for now.
*
* We don't have a do_deferred_work down this pathway, on the
* principle that we didn't *really* affect shard state (we
* tweaked the stats, but our tweaks weren't really accurate).
*/
psset_update_end(&shard->psset, ps);
edata_cache_fast_put(tsdn, &shard->ecf, edata);
*oom = true;
return NULL;
}
hpa_update_purge_hugify_eligibility(tsdn, shard, ps);
psset_update_end(&shard->psset, ps);
return edata;
}
static size_t
hpa_try_alloc_batch_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
bool *oom, size_t nallocs, edata_list_active_t *results,
bool *deferred_work_generated) {
malloc_mutex_lock(tsdn, &shard->mtx);
size_t nsuccess = 0;
for (; nsuccess < nallocs; nsuccess++) {
edata_t *edata = hpa_try_alloc_one_no_grow(tsdn, shard, size,
oom);
if (edata == NULL) {
break;
}
edata_list_active_append(results, edata);
}
hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false);
*deferred_work_generated = hpa_shard_has_deferred_work(tsdn, shard);
malloc_mutex_unlock(tsdn, &shard->mtx);
return nsuccess;
}
static size_t
hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
size_t nallocs, edata_list_active_t *results,
bool *deferred_work_generated) {
assert(size <= shard->opts.slab_max_alloc);
bool oom = false;
size_t nsuccess = hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
nallocs, results, deferred_work_generated);
if (nsuccess == nallocs || oom) {
return nsuccess;
}
/*
* We didn't OOM, but weren't able to fill everything requested of us;
* try to grow.
*/
malloc_mutex_lock(tsdn, &shard->grow_mtx);
/*
* Check for grow races; maybe some earlier thread expanded the psset
* in between when we dropped the main mutex and grabbed the grow mutex.
*/
nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
nallocs - nsuccess, results, deferred_work_generated);
if (nsuccess == nallocs || oom) {
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
return nsuccess;
}
/*
* Note that we don't hold shard->mtx here (while growing);
* deallocations (and allocations of smaller sizes) may still succeed
* while we're doing this potentially expensive system call.
*/
hpdata_t *ps = hpa_central_extract(tsdn, shard->central, size, &oom);
if (ps == NULL) {
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
return nsuccess;
}
/*
* We got the pageslab; allocate from it. This does an unlock followed
* by a lock on the same mutex, and holds the grow mutex while doing
* deferred work, but this is an uncommon path; the simplicity is worth
* it.
*/
malloc_mutex_lock(tsdn, &shard->mtx);
psset_insert(&shard->psset, ps);
malloc_mutex_unlock(tsdn, &shard->mtx);
nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
nallocs - nsuccess, results, deferred_work_generated);
/*
* Drop grow_mtx before doing deferred work; other threads blocked on it
* should be allowed to proceed while we're working.
*/
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
return nsuccess;
}
static hpa_shard_t *
hpa_from_pai(pai_t *self) {
assert(self->alloc = &hpa_alloc);
assert(self->expand = &hpa_expand);
assert(self->shrink = &hpa_shrink);
assert(self->dalloc = &hpa_dalloc);
return (hpa_shard_t *)self;
}
static size_t
hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
edata_list_active_t *results, bool *deferred_work_generated) {
assert(nallocs > 0);
assert((size & PAGE_MASK) == 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
hpa_shard_t *shard = hpa_from_pai(self);
if (size > shard->opts.slab_max_alloc) {
return 0;
}
size_t nsuccess = hpa_alloc_batch_psset(tsdn, shard, size, nallocs,
results, deferred_work_generated);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
/*
* Guard the sanity checks with config_debug because the loop cannot be
* proven non-circular by the compiler, even if everything within the
* loop is optimized away.
*/
if (config_debug) {
edata_t *edata;
ql_foreach(edata, &results->head, ql_link_active) {
emap_assert_mapped(tsdn, shard->emap, edata);
assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
assert(edata_state_get(edata) == extent_state_active);
assert(edata_arena_ind_get(edata) == shard->ind);
assert(edata_szind_get_maybe_invalid(edata) ==
SC_NSIZES);
assert(!edata_slab_get(edata));
assert(edata_committed_get(edata));
assert(edata_base_get(edata) == edata_addr_get(edata));
assert(edata_base_get(edata) != NULL);
}
}
return nsuccess;
}
static edata_t *
hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
assert((size & PAGE_MASK) == 0);
assert(!guarded);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
/* We don't handle alignment or zeroing for now. */
if (alignment > PAGE || zero) {
return NULL;
}
/*
* An alloc with alignment == PAGE and zero == false is equivalent to a
* batch alloc of 1. Just do that, so we can share code.
*/
edata_list_active_t results;
edata_list_active_init(&results);
size_t nallocs = hpa_alloc_batch(tsdn, self, size, /* nallocs */ 1,
&results, deferred_work_generated);
assert(nallocs == 0 || nallocs == 1);
edata_t *edata = edata_list_active_first(&results);
return edata;
}
static bool
hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool zero, bool *deferred_work_generated) {
/* Expand not yet supported. */
return true;
}
static bool
hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool *deferred_work_generated) {
/* Shrink not yet supported. */
return true;
}
static void
hpa_dalloc_prepare_unlocked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
assert(edata_state_get(edata) == extent_state_active);
assert(edata_arena_ind_get(edata) == shard->ind);
assert(edata_szind_get_maybe_invalid(edata) == SC_NSIZES);
assert(edata_committed_get(edata));
assert(edata_base_get(edata) != NULL);
/*
* Another thread shouldn't be trying to touch the metadata of an
* allocation being freed. The one exception is a merge attempt from a
* lower-addressed PAC extent; in this case we have a nominal race on
* the edata metadata bits, but in practice the fact that the PAI bits
* are different will prevent any further access. The race is bad, but
* benign in practice, and the long term plan is to track enough state
* in the rtree to prevent these merge attempts in the first place.
*/
edata_addr_set(edata, edata_base_get(edata));
edata_zeroed_set(edata, false);
emap_deregister_boundary(tsdn, shard->emap, edata);
}
static void
hpa_dalloc_locked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
/*
* Release the metadata early, to avoid having to remember to do it
* while we're also doing tricky purging logic. First, we need to grab
* a few bits of metadata from it.
*
* Note that the shard mutex protects ps's metadata too; it wouldn't be
* correct to try to read most information out of it without the lock.
*/
hpdata_t *ps = edata_ps_get(edata);
/* Currently, all edatas come from pageslabs. */
assert(ps != NULL);
void *unreserve_addr = edata_addr_get(edata);
size_t unreserve_size = edata_size_get(edata);
edata_cache_fast_put(tsdn, &shard->ecf, edata);
psset_update_begin(&shard->psset, ps);
hpdata_unreserve(ps, unreserve_addr, unreserve_size);
hpa_update_purge_hugify_eligibility(tsdn, shard, ps);
psset_update_end(&shard->psset, ps);
}
static void
hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
bool *deferred_work_generated) {
hpa_shard_t *shard = hpa_from_pai(self);
edata_t *edata;
ql_foreach(edata, &list->head, ql_link_active) {
hpa_dalloc_prepare_unlocked(tsdn, shard, edata);
}
malloc_mutex_lock(tsdn, &shard->mtx);
/* Now, remove from the list. */
while ((edata = edata_list_active_first(list)) != NULL) {
edata_list_active_remove(list, edata);
hpa_dalloc_locked(tsdn, shard, edata);
}
hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false);
*deferred_work_generated =
hpa_shard_has_deferred_work(tsdn, shard);
malloc_mutex_unlock(tsdn, &shard->mtx);
}
static void
hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated) {
assert(!edata_guarded_get(edata));
/* Just a dalloc_batch of size 1; this lets us share logic. */
edata_list_active_t dalloc_list;
edata_list_active_init(&dalloc_list);
edata_list_active_append(&dalloc_list, edata);
hpa_dalloc_batch(tsdn, self, &dalloc_list, deferred_work_generated);
}
/*
* Calculate time until either purging or hugification ought to happen.
* Called by background threads.
*/
static uint64_t
hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
hpa_shard_t *shard = hpa_from_pai(self);
uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX;
malloc_mutex_lock(tsdn, &shard->mtx);
hpdata_t *to_hugify = psset_pick_hugify(&shard->psset);
if (to_hugify != NULL) {
nstime_t time_hugify_allowed =
hpdata_time_hugify_allowed(to_hugify);
uint64_t since_hugify_allowed_ms =
shard->central->hooks.ms_since(&time_hugify_allowed);
/*
* If not enough time has passed since hugification was allowed,
* sleep for the rest.
*/
if (since_hugify_allowed_ms < shard->opts.hugify_delay_ms) {
time_ns = shard->opts.hugify_delay_ms -
since_hugify_allowed_ms;
time_ns *= 1000 * 1000;
} else {
malloc_mutex_unlock(tsdn, &shard->mtx);
return BACKGROUND_THREAD_DEFERRED_MIN;
}
}
if (hpa_should_purge(tsdn, shard)) {
/*
* If we haven't purged before, no need to check interval
* between purges. Simply purge as soon as possible.
*/
if (shard->stats.npurge_passes == 0) {
malloc_mutex_unlock(tsdn, &shard->mtx);
return BACKGROUND_THREAD_DEFERRED_MIN;
}
uint64_t since_last_purge_ms = shard->central->hooks.ms_since(
&shard->last_purge);
if (since_last_purge_ms < shard->opts.min_purge_interval_ms) {
uint64_t until_purge_ns;
until_purge_ns = shard->opts.min_purge_interval_ms -
since_last_purge_ms;
until_purge_ns *= 1000 * 1000;
if (until_purge_ns < time_ns) {
time_ns = until_purge_ns;
}
} else {
time_ns = BACKGROUND_THREAD_DEFERRED_MIN;
}
}
malloc_mutex_unlock(tsdn, &shard->mtx);
return time_ns;
}
void
hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
malloc_mutex_lock(tsdn, &shard->mtx);
edata_cache_fast_disable(tsdn, &shard->ecf);
malloc_mutex_unlock(tsdn, &shard->mtx);
}
static void
hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) {
assert(bin_stats->npageslabs == 0);
assert(bin_stats->nactive == 0);
}
static void
hpa_assert_empty(tsdn_t *tsdn, hpa_shard_t *shard, psset_t *psset) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
for (int huge = 0; huge <= 1; huge++) {
hpa_shard_assert_stats_empty(&psset->stats.full_slabs[huge]);
for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
hpa_shard_assert_stats_empty(
&psset->stats.nonfull_slabs[i][huge]);
}
}
}
void
hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
/*
* By the time we're here, the arena code should have dalloc'd all the
* active extents, which means we should have eventually evicted
* everything from the psset, so it shouldn't be able to serve even a
* 1-page allocation.
*/
if (config_debug) {
malloc_mutex_lock(tsdn, &shard->mtx);
hpa_assert_empty(tsdn, shard, &shard->psset);
malloc_mutex_unlock(tsdn, &shard->mtx);
}
hpdata_t *ps;
while ((ps = psset_pick_alloc(&shard->psset, PAGE)) != NULL) {
/* There should be no allocations anywhere. */
assert(hpdata_empty(ps));
psset_remove(&shard->psset, ps);
shard->central->hooks.unmap(hpdata_addr_get(ps), HUGEPAGE);
}
}
void
hpa_shard_set_deferral_allowed(tsdn_t *tsdn, hpa_shard_t *shard,
bool deferral_allowed) {
hpa_do_consistency_checks(shard);
malloc_mutex_lock(tsdn, &shard->mtx);
bool deferral_previously_allowed = shard->opts.deferral_allowed;
shard->opts.deferral_allowed = deferral_allowed;
if (deferral_previously_allowed && !deferral_allowed) {
hpa_shard_maybe_do_deferred_work(tsdn, shard,
/* forced */ true);
}
malloc_mutex_unlock(tsdn, &shard->mtx);
}
void
hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
malloc_mutex_lock(tsdn, &shard->mtx);
hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ true);
malloc_mutex_unlock(tsdn, &shard->mtx);
}
void
hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
malloc_mutex_prefork(tsdn, &shard->grow_mtx);
}
void
hpa_shard_prefork4(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
malloc_mutex_prefork(tsdn, &shard->mtx);
}
void
hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
malloc_mutex_postfork_parent(tsdn, &shard->grow_mtx);
malloc_mutex_postfork_parent(tsdn, &shard->mtx);
}
void
hpa_shard_postfork_child(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
malloc_mutex_postfork_child(tsdn, &shard->grow_mtx);
malloc_mutex_postfork_child(tsdn, &shard->mtx);
}
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/hpa_hooks.h"
static void *hpa_hooks_map(size_t size);
static void hpa_hooks_unmap(void *ptr, size_t size);
static void hpa_hooks_purge(void *ptr, size_t size);
static void hpa_hooks_hugify(void *ptr, size_t size);
static void hpa_hooks_dehugify(void *ptr, size_t size);
static void hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading);
static uint64_t hpa_hooks_ms_since(nstime_t *past_nstime);
hpa_hooks_t hpa_hooks_default = {
&hpa_hooks_map,
&hpa_hooks_unmap,
&hpa_hooks_purge,
&hpa_hooks_hugify,
&hpa_hooks_dehugify,
&hpa_hooks_curtime,
&hpa_hooks_ms_since
};
static void *
hpa_hooks_map(size_t size) {
bool commit = true;
return pages_map(NULL, size, HUGEPAGE, &commit);
}
static void
hpa_hooks_unmap(void *ptr, size_t size) {
pages_unmap(ptr, size);
}
static void
hpa_hooks_purge(void *ptr, size_t size) {
pages_purge_forced(ptr, size);
}
static void
hpa_hooks_hugify(void *ptr, size_t size) {
bool err = pages_huge(ptr, size);
(void)err;
}
static void
hpa_hooks_dehugify(void *ptr, size_t size) {
bool err = pages_nohuge(ptr, size);
(void)err;
}
static void
hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading) {
if (first_reading) {
nstime_init_zero(r_nstime);
}
nstime_update(r_nstime);
}
static uint64_t
hpa_hooks_ms_since(nstime_t *past_nstime) {
return nstime_ns_since(past_nstime) / 1000 / 1000;
}
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/hpdata.h"
static int
hpdata_age_comp(const hpdata_t *a, const hpdata_t *b) {
uint64_t a_age = hpdata_age_get(a);
uint64_t b_age = hpdata_age_get(b);
/*
* hpdata ages are operation counts in the psset; no two should be the
* same.
*/
assert(a_age != b_age);
return (a_age > b_age) - (a_age < b_age);
}
ph_gen(, hpdata_age_heap, hpdata_t, age_link, hpdata_age_comp)
void
hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) {
hpdata_addr_set(hpdata, addr);
hpdata_age_set(hpdata, age);
hpdata->h_huge = false;
hpdata->h_alloc_allowed = true;
hpdata->h_in_psset_alloc_container = false;
hpdata->h_purge_allowed = false;
hpdata->h_hugify_allowed = false;
hpdata->h_in_psset_hugify_container = false;
hpdata->h_mid_purge = false;
hpdata->h_mid_hugify = false;
hpdata->h_updating = false;
hpdata->h_in_psset = false;
hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES);
hpdata->h_nactive = 0;
fb_init(hpdata->active_pages, HUGEPAGE_PAGES);
hpdata->h_ntouched = 0;
fb_init(hpdata->touched_pages, HUGEPAGE_PAGES);
hpdata_assert_consistent(hpdata);
}
void *
hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) {
hpdata_assert_consistent(hpdata);
/*
* This is a metadata change; the hpdata should therefore either not be
* in the psset, or should have explicitly marked itself as being
* mid-update.
*/
assert(!hpdata->h_in_psset || hpdata->h_updating);
assert(hpdata->h_alloc_allowed);
assert((sz & PAGE_MASK) == 0);
size_t npages = sz >> LG_PAGE;
assert(npages <= hpdata_longest_free_range_get(hpdata));
size_t result;
size_t start = 0;
/*
* These are dead stores, but the compiler will issue warnings on them
* since it can't tell statically that found is always true below.
*/
size_t begin = 0;
size_t len = 0;
size_t largest_unchosen_range = 0;
while (true) {
bool found = fb_urange_iter(hpdata->active_pages,
HUGEPAGE_PAGES, start, &begin, &len);
/*
* A precondition to this function is that hpdata must be able
* to serve the allocation.
*/
assert(found);
assert(len <= hpdata_longest_free_range_get(hpdata));
if (len >= npages) {
/*
* We use first-fit within the page slabs; this gives
* bounded worst-case fragmentation within a slab. It's
* not necessarily right; we could experiment with
* various other options.
*/
break;
}
if (len > largest_unchosen_range) {
largest_unchosen_range = len;
}
start = begin + len;
}
/* We found a range; remember it. */
result = begin;
fb_set_range(hpdata->active_pages, HUGEPAGE_PAGES, begin, npages);
hpdata->h_nactive += npages;
/*
* We might be about to dirty some memory for the first time; update our
* count if so.
*/
size_t new_dirty = fb_ucount(hpdata->touched_pages, HUGEPAGE_PAGES,
result, npages);
fb_set_range(hpdata->touched_pages, HUGEPAGE_PAGES, result, npages);
hpdata->h_ntouched += new_dirty;
/*
* If we allocated out of a range that was the longest in the hpdata, it
* might be the only one of that size and we'll have to adjust the
* metadata.
*/
if (len == hpdata_longest_free_range_get(hpdata)) {
start = begin + npages;
while (start < HUGEPAGE_PAGES) {
bool found = fb_urange_iter(hpdata->active_pages,
HUGEPAGE_PAGES, start, &begin, &len);
if (!found) {
break;
}
assert(len <= hpdata_longest_free_range_get(hpdata));
if (len == hpdata_longest_free_range_get(hpdata)) {
largest_unchosen_range = len;
break;
}
if (len > largest_unchosen_range) {
largest_unchosen_range = len;
}
start = begin + len;
}
hpdata_longest_free_range_set(hpdata, largest_unchosen_range);
}
hpdata_assert_consistent(hpdata);
return (void *)(
(uintptr_t)hpdata_addr_get(hpdata) + (result << LG_PAGE));
}
void
hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {
hpdata_assert_consistent(hpdata);
/* See the comment in reserve. */
assert(!hpdata->h_in_psset || hpdata->h_updating);
assert(((uintptr_t)addr & PAGE_MASK) == 0);
assert((sz & PAGE_MASK) == 0);
size_t begin = ((uintptr_t)addr - (uintptr_t)hpdata_addr_get(hpdata))
>> LG_PAGE;
assert(begin < HUGEPAGE_PAGES);
size_t npages = sz >> LG_PAGE;
size_t old_longest_range = hpdata_longest_free_range_get(hpdata);
fb_unset_range(hpdata->active_pages, HUGEPAGE_PAGES, begin, npages);
/* We might have just created a new, larger range. */
size_t new_begin = (fb_fls(hpdata->active_pages, HUGEPAGE_PAGES,
begin) + 1);
size_t new_end = fb_ffs(hpdata->active_pages, HUGEPAGE_PAGES,
begin + npages - 1);
size_t new_range_len = new_end - new_begin;
if (new_range_len > old_longest_range) {
hpdata_longest_free_range_set(hpdata, new_range_len);
}
hpdata->h_nactive -= npages;
hpdata_assert_consistent(hpdata);
}
size_t
hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) {
hpdata_assert_consistent(hpdata);
/*
* See the comment below; we might purge any inactive extent, so it's
* unsafe for any other thread to turn any inactive extent active while
* we're operating on it.
*/
assert(!hpdata_alloc_allowed_get(hpdata));
purge_state->npurged = 0;
purge_state->next_purge_search_begin = 0;
/*
* Initialize to_purge.
*
* It's possible to end up in situations where two dirty extents are
* separated by a retained extent:
* - 1 page allocated.
* - 1 page allocated.
* - 1 pages allocated.
*
* If the middle page is freed and purged, and then the first and third
* pages are freed, and then another purge pass happens, the hpdata
* looks like this:
* - 1 page dirty.
* - 1 page retained.
* - 1 page dirty.
*
* But it's safe to do a single 3-page purge.
*
* We do this by first computing the dirty pages, and then filling in
* any gaps by extending each range in the dirty bitmap to extend until
* the next active page. This purges more pages, but the expensive part
* of purging is the TLB shootdowns, rather than the kernel state
* tracking; doing a little bit more of the latter is fine if it saves
* us from doing some of the former.
*/
/*
* The dirty pages are those that are touched but not active. Note that
* in a normal-ish case, HUGEPAGE_PAGES is something like 512 and the
* fb_group_t is 64 bits, so this is 64 bytes, spread across 8
* fb_group_ts.
*/
fb_group_t dirty_pages[FB_NGROUPS(HUGEPAGE_PAGES)];
fb_init(dirty_pages, HUGEPAGE_PAGES);
fb_bit_not(dirty_pages, hpdata->active_pages, HUGEPAGE_PAGES);
fb_bit_and(dirty_pages, dirty_pages, hpdata->touched_pages,
HUGEPAGE_PAGES);
fb_init(purge_state->to_purge, HUGEPAGE_PAGES);
size_t next_bit = 0;
while (next_bit < HUGEPAGE_PAGES) {
size_t next_dirty = fb_ffs(dirty_pages, HUGEPAGE_PAGES,
next_bit);
/* Recall that fb_ffs returns nbits if no set bit is found. */
if (next_dirty == HUGEPAGE_PAGES) {
break;
}
size_t next_active = fb_ffs(hpdata->active_pages,
HUGEPAGE_PAGES, next_dirty);
/*
* Don't purge past the end of the dirty extent, into retained
* pages. This helps the kernel a tiny bit, but honestly it's
* mostly helpful for testing (where we tend to write test cases
* that think in terms of the dirty ranges).
*/
ssize_t last_dirty = fb_fls(dirty_pages, HUGEPAGE_PAGES,
next_active - 1);
assert(last_dirty >= 0);
assert((size_t)last_dirty >= next_dirty);
assert((size_t)last_dirty - next_dirty + 1 <= HUGEPAGE_PAGES);
fb_set_range(purge_state->to_purge, HUGEPAGE_PAGES, next_dirty,
last_dirty - next_dirty + 1);
next_bit = next_active + 1;
}
/* We should purge, at least, everything dirty. */
size_t ndirty = hpdata->h_ntouched - hpdata->h_nactive;
purge_state->ndirty_to_purge = ndirty;
assert(ndirty <= fb_scount(
purge_state->to_purge, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES));
assert(ndirty == fb_scount(dirty_pages, HUGEPAGE_PAGES, 0,
HUGEPAGE_PAGES));
hpdata_assert_consistent(hpdata);
return ndirty;
}
bool
hpdata_purge_next(hpdata_t *hpdata, hpdata_purge_state_t *purge_state,
void **r_purge_addr, size_t *r_purge_size) {
/*
* Note that we don't have a consistency check here; we're accessing
* hpdata without synchronization, and therefore have no right to expect
* a consistent state.
*/
assert(!hpdata_alloc_allowed_get(hpdata));
if (purge_state->next_purge_search_begin == HUGEPAGE_PAGES) {
return false;
}
size_t purge_begin;
size_t purge_len;
bool found_range = fb_srange_iter(purge_state->to_purge, HUGEPAGE_PAGES,
purge_state->next_purge_search_begin, &purge_begin, &purge_len);
if (!found_range) {
return false;
}
*r_purge_addr = (void *)(
(uintptr_t)hpdata_addr_get(hpdata) + purge_begin * PAGE);
*r_purge_size = purge_len * PAGE;
purge_state->next_purge_search_begin = purge_begin + purge_len;
purge_state->npurged += purge_len;
assert(purge_state->npurged <= HUGEPAGE_PAGES);
return true;
}
void
hpdata_purge_end(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) {
assert(!hpdata_alloc_allowed_get(hpdata));
hpdata_assert_consistent(hpdata);
/* See the comment in reserve. */
assert(!hpdata->h_in_psset || hpdata->h_updating);
assert(purge_state->npurged == fb_scount(purge_state->to_purge,
HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES));
assert(purge_state->npurged >= purge_state->ndirty_to_purge);
fb_bit_not(purge_state->to_purge, purge_state->to_purge,
HUGEPAGE_PAGES);
fb_bit_and(hpdata->touched_pages, hpdata->touched_pages,
purge_state->to_purge, HUGEPAGE_PAGES);
assert(hpdata->h_ntouched >= purge_state->ndirty_to_purge);
hpdata->h_ntouched -= purge_state->ndirty_to_purge;
hpdata_assert_consistent(hpdata);
}
void
hpdata_hugify(hpdata_t *hpdata) {
hpdata_assert_consistent(hpdata);
hpdata->h_huge = true;
fb_set_range(hpdata->touched_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES);
hpdata->h_ntouched = HUGEPAGE_PAGES;
hpdata_assert_consistent(hpdata);
}
void
hpdata_dehugify(hpdata_t *hpdata) {
hpdata_assert_consistent(hpdata);
hpdata->h_huge = false;
hpdata_assert_consistent(hpdata);
}
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
void
inspect_extent_util_stats_get(tsdn_t *tsdn, const void *ptr, size_t *nfree,
size_t *nregs, size_t *size) {
assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL);
const edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
if (unlikely(edata == NULL)) {
*nfree = *nregs = *size = 0;
return;
}
*size = edata_size_get(edata);
if (!edata_slab_get(edata)) {
*nfree = 0;
*nregs = 1;
} else {
*nfree = edata_nfree_get(edata);
*nregs = bin_infos[edata_szind_get(edata)].nregs;
assert(*nfree <= *nregs);
assert(*nfree * edata_usize_get(edata) <= *size);
}
}
void
inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
size_t *nfree, size_t *nregs, size_t *size, size_t *bin_nfree,
size_t *bin_nregs, void **slabcur_addr) {
assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL
&& bin_nfree != NULL && bin_nregs != NULL && slabcur_addr != NULL);
const edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
if (unlikely(edata == NULL)) {
*nfree = *nregs = *size = *bin_nfree = *bin_nregs = 0;
*slabcur_addr = NULL;
return;
}
*size = edata_size_get(edata);
if (!edata_slab_get(edata)) {
*nfree = *bin_nfree = *bin_nregs = 0;
*nregs = 1;
*slabcur_addr = NULL;
return;
}
*nfree = edata_nfree_get(edata);
const szind_t szind = edata_szind_get(edata);
*nregs = bin_infos[szind].nregs;
assert(*nfree <= *nregs);
assert(*nfree * edata_usize_get(edata) <= *size);
arena_t *arena = (arena_t *)atomic_load_p(
&arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
assert(arena != NULL);
const unsigned binshard = edata_binshard_get(edata);
bin_t *bin = arena_get_bin(arena, szind, binshard);
malloc_mutex_lock(tsdn, &bin->lock);
if (config_stats) {
*bin_nregs = *nregs * bin->stats.curslabs;
assert(*bin_nregs >= bin->stats.curregs);
*bin_nfree = *bin_nregs - bin->stats.curregs;
} else {
*bin_nfree = *bin_nregs = 0;
}
edata_t *slab;
if (bin->slabcur != NULL) {
slab = bin->slabcur;
} else {
slab = edata_heap_first(&bin->slabs_nonfull);
}
*slabcur_addr = slab != NULL ? edata_addr_get(slab) : NULL;
malloc_mutex_unlock(tsdn, &bin->lock);
}
......@@ -4,20 +4,26 @@
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/buf_writer.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/fxp.h"
#include "jemalloc/internal/san.h"
#include "jemalloc/internal/hook.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/log.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/spin.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/thread_event.h"
#include "jemalloc/internal/util.h"
/******************************************************************************/
......@@ -29,6 +35,29 @@ const char *je_malloc_conf
JEMALLOC_ATTR(weak)
#endif
;
/*
* The usual rule is that the closer to runtime you are, the higher priority
* your configuration settings are (so the jemalloc config options get lower
* priority than the per-binary setting, which gets lower priority than the /etc
* setting, which gets lower priority than the environment settings).
*
* But it's a fairly common use case in some testing environments for a user to
* be able to control the binary, but nothing else (e.g. a performancy canary
* uses the production OS and environment variables, but can run any binary in
* those circumstances). For these use cases, it's handy to have an in-binary
* mechanism for overriding environment variable settings, with the idea that if
* the results are positive they get promoted to the official settings, and
* moved from the binary to the environment variable.
*
* We don't actually want this to be widespread, so we'll give it a silly name
* and not mention it in headers or documentation.
*/
const char *je_malloc_conf_2_conf_harder
#ifndef _WIN32
JEMALLOC_ATTR(weak)
#endif
;
bool opt_abort =
#ifdef JEMALLOC_DEBUG
true
......@@ -66,16 +95,73 @@ bool opt_junk_free =
false
#endif
;
bool opt_trust_madvise =
#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
false
#else
true
#endif
;
bool opt_cache_oblivious =
#ifdef JEMALLOC_CACHE_OBLIVIOUS
true
#else
false
#endif
;
zero_realloc_action_t opt_zero_realloc_action =
#ifdef JEMALLOC_ZERO_REALLOC_DEFAULT_FREE
zero_realloc_action_free
#else
zero_realloc_action_alloc
#endif
;
atomic_zu_t zero_realloc_count = ATOMIC_INIT(0);
const char *zero_realloc_mode_names[] = {
"alloc",
"free",
"abort",
};
/*
* These are the documented values for junk fill debugging facilities -- see the
* man page.
*/
static const uint8_t junk_alloc_byte = 0xa5;
static const uint8_t junk_free_byte = 0x5a;
static void default_junk_alloc(void *ptr, size_t usize) {
memset(ptr, junk_alloc_byte, usize);
}
static void default_junk_free(void *ptr, size_t usize) {
memset(ptr, junk_free_byte, usize);
}
void (*junk_alloc_callback)(void *ptr, size_t size) = &default_junk_alloc;
void (*junk_free_callback)(void *ptr, size_t size) = &default_junk_free;
bool opt_utrace = false;
bool opt_xmalloc = false;
bool opt_experimental_infallible_new = false;
bool opt_zero = false;
unsigned opt_narenas = 0;
fxp_t opt_narenas_ratio = FXP_INIT_INT(4);
unsigned ncpus;
/* Protects arenas initialization. */
malloc_mutex_t arenas_lock;
/* The global hpa, and whether it's on. */
bool opt_hpa = false;
hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT;
sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT;
/*
* Arenas that are used to service external requests. Not all elements of the
* arenas array are necessarily used; arenas are created lazily as needed.
......@@ -94,13 +180,7 @@ static arena_t *a0; /* arenas[0]. */
unsigned narenas_auto;
unsigned manual_arena_base;
typedef enum {
malloc_init_uninitialized = 3,
malloc_init_a0_initialized = 2,
malloc_init_recursible = 1,
malloc_init_initialized = 0 /* Common case --> jnz. */
} malloc_init_t;
static malloc_init_t malloc_init_state = malloc_init_uninitialized;
malloc_init_t malloc_init_state = malloc_init_uninitialized;
/* False should be the common case. Set to true to trigger initialization. */
bool malloc_slow = true;
......@@ -180,7 +260,7 @@ typedef struct {
ut.p = (a); \
ut.s = (b); \
ut.r = (c); \
utrace(&ut, sizeof(ut)); \
UTRACE_CALL(&ut, sizeof(ut)); \
errno = utrace_serrno; \
} \
} while (0)
......@@ -205,11 +285,6 @@ static bool malloc_init_hard(void);
* Begin miscellaneous support functions.
*/
bool
malloc_initialized(void) {
return (malloc_init_state == malloc_init_initialized);
}
JEMALLOC_ALWAYS_INLINE bool
malloc_init_a0(void) {
if (unlikely(malloc_init_state == malloc_init_uninitialized)) {
......@@ -257,7 +332,7 @@ a0dalloc(void *ptr) {
}
/*
* FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive
* FreeBSD's libc uses the bootstrap_*() functions in bootstrap-sensitive
* situations that cannot tolerate TLS variable access (TLS allocation and very
* early internal data structure initialization).
*/
......@@ -315,7 +390,7 @@ narenas_total_get(void) {
/* Create a new arena and insert it into the arenas array at index ind. */
static arena_t *
arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
arena_init_locked(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
arena_t *arena;
assert(ind <= narenas_total_get());
......@@ -337,7 +412,7 @@ arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
}
/* Actually initialize the arena. */
arena = arena_new(tsdn, ind, extent_hooks);
arena = arena_new(tsdn, ind, config);
return arena;
}
......@@ -361,11 +436,11 @@ arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) {
}
arena_t *
arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
arena_t *arena;
malloc_mutex_lock(tsdn, &arenas_lock);
arena = arena_init_locked(tsdn, ind, extent_hooks);
arena = arena_init_locked(tsdn, ind, config);
malloc_mutex_unlock(tsdn, &arenas_lock);
arena_new_create_background_thread(tsdn, ind);
......@@ -394,14 +469,19 @@ arena_bind(tsd_t *tsd, unsigned ind, bool internal) {
}
void
arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) {
arena_t *oldarena, *newarena;
arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
assert(oldarena != NULL);
assert(newarena != NULL);
oldarena = arena_get(tsd_tsdn(tsd), oldind, false);
newarena = arena_get(tsd_tsdn(tsd), newind, false);
arena_nthreads_dec(oldarena, false);
arena_nthreads_inc(newarena, false);
tsd_arena_set(tsd, newarena);
if (arena_nthreads_get(oldarena, false) == 0) {
/* Purge if the old arena has no associated threads anymore. */
arena_decay(tsd_tsdn(tsd), oldarena,
/* is_background_thread */ false, /* all */ true);
}
}
static void
......@@ -418,82 +498,6 @@ arena_unbind(tsd_t *tsd, unsigned ind, bool internal) {
}
}
arena_tdata_t *
arena_tdata_get_hard(tsd_t *tsd, unsigned ind) {
arena_tdata_t *tdata, *arenas_tdata_old;
arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd);
unsigned narenas_tdata_old, i;
unsigned narenas_tdata = tsd_narenas_tdata_get(tsd);
unsigned narenas_actual = narenas_total_get();
/*
* Dissociate old tdata array (and set up for deallocation upon return)
* if it's too small.
*/
if (arenas_tdata != NULL && narenas_tdata < narenas_actual) {
arenas_tdata_old = arenas_tdata;
narenas_tdata_old = narenas_tdata;
arenas_tdata = NULL;
narenas_tdata = 0;
tsd_arenas_tdata_set(tsd, arenas_tdata);
tsd_narenas_tdata_set(tsd, narenas_tdata);
} else {
arenas_tdata_old = NULL;
narenas_tdata_old = 0;
}
/* Allocate tdata array if it's missing. */
if (arenas_tdata == NULL) {
bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd);
narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1;
if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) {
*arenas_tdata_bypassp = true;
arenas_tdata = (arena_tdata_t *)a0malloc(
sizeof(arena_tdata_t) * narenas_tdata);
*arenas_tdata_bypassp = false;
}
if (arenas_tdata == NULL) {
tdata = NULL;
goto label_return;
}
assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp);
tsd_arenas_tdata_set(tsd, arenas_tdata);
tsd_narenas_tdata_set(tsd, narenas_tdata);
}
/*
* Copy to tdata array. It's possible that the actual number of arenas
* has increased since narenas_total_get() was called above, but that
* causes no correctness issues unless two threads concurrently execute
* the arenas.create mallctl, which we trust mallctl synchronization to
* prevent.
*/
/* Copy/initialize tickers. */
for (i = 0; i < narenas_actual; i++) {
if (i < narenas_tdata_old) {
ticker_copy(&arenas_tdata[i].decay_ticker,
&arenas_tdata_old[i].decay_ticker);
} else {
ticker_init(&arenas_tdata[i].decay_ticker,
DECAY_NTICKS_PER_UPDATE);
}
}
if (narenas_tdata > narenas_actual) {
memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t)
* (narenas_tdata - narenas_actual));
}
/* Read the refreshed tdata array. */
tdata = &arenas_tdata[ind];
label_return:
if (arenas_tdata_old != NULL) {
a0dalloc(arenas_tdata_old);
}
return tdata;
}
/* Slow path, called only by arena_choose(). */
arena_t *
arena_choose_hard(tsd_t *tsd, bool internal) {
......@@ -576,8 +580,7 @@ arena_choose_hard(tsd_t *tsd, bool internal) {
/* Initialize a new arena. */
choose[j] = first_null;
arena = arena_init_locked(tsd_tsdn(tsd),
choose[j],
(extent_hooks_t *)&extent_hooks_default);
choose[j], &arena_config_default);
if (arena == NULL) {
malloc_mutex_unlock(tsd_tsdn(tsd),
&arenas_lock);
......@@ -629,20 +632,6 @@ arena_cleanup(tsd_t *tsd) {
}
}
void
arenas_tdata_cleanup(tsd_t *tsd) {
arena_tdata_t *arenas_tdata;
/* Prevent tsd->arenas_tdata from being (re)created. */
*tsd_arenas_tdata_bypassp_get(tsd) = true;
arenas_tdata = tsd_arenas_tdata_get(tsd);
if (arenas_tdata != NULL) {
tsd_arenas_tdata_set(tsd, NULL);
a0dalloc(arenas_tdata);
}
}
static void
stats_print_atexit(void) {
if (config_stats) {
......@@ -661,11 +650,13 @@ stats_print_atexit(void) {
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena = arena_get(tsdn, i, false);
if (arena != NULL) {
tcache_t *tcache;
tcache_slow_t *tcache_slow;
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
ql_foreach(tcache, &arena->tcache_ql, link) {
tcache_stats_merge(tsdn, tcache, arena);
ql_foreach(tcache_slow, &arena->tcache_ql,
link) {
tcache_stats_merge(tsdn,
tcache_slow->tcache, arena);
}
malloc_mutex_unlock(tsdn,
&arena->tcache_ql_mtx);
......@@ -730,18 +721,28 @@ malloc_ncpus(void) {
SYSTEM_INFO si;
GetSystemInfo(&si);
result = si.dwNumberOfProcessors;
#elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT)
#elif defined(CPU_COUNT)
/*
* glibc >= 2.6 has the CPU_COUNT macro.
*
* glibc's sysconf() uses isspace(). glibc allocates for the first time
* *before* setting up the isspace tables. Therefore we need a
* different method to get the number of CPUs.
*
* The getaffinity approach is also preferred when only a subset of CPUs
* is available, to avoid using more arenas than necessary.
*/
{
# if defined(__FreeBSD__) || defined(__DragonFly__)
cpuset_t set;
# else
cpu_set_t set;
# endif
# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
sched_getaffinity(0, sizeof(set), &set);
# else
pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
# endif
result = CPU_COUNT(&set);
}
#else
......@@ -750,9 +751,47 @@ malloc_ncpus(void) {
return ((result == -1) ? 1 : (unsigned)result);
}
/*
* Ensure that number of CPUs is determistinc, i.e. it is the same based on:
* - sched_getaffinity()
* - _SC_NPROCESSORS_ONLN
* - _SC_NPROCESSORS_CONF
* Since otherwise tricky things is possible with percpu arenas in use.
*/
static bool
malloc_cpu_count_is_deterministic()
{
#ifdef _WIN32
return true;
#else
long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN);
long cpu_conf = sysconf(_SC_NPROCESSORS_CONF);
if (cpu_onln != cpu_conf) {
return false;
}
# if defined(CPU_COUNT)
# if defined(__FreeBSD__) || defined(__DragonFly__)
cpuset_t set;
# else
cpu_set_t set;
# endif /* __FreeBSD__ */
# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
sched_getaffinity(0, sizeof(set), &set);
# else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */
pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
# endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */
long cpu_affinity = CPU_COUNT(&set);
if (cpu_affinity != cpu_conf) {
return false;
}
# endif /* CPU_COUNT */
return true;
#endif
}
static void
init_opt_stats_print_opts(const char *v, size_t vlen) {
size_t opts_len = strlen(opt_stats_print_opts);
init_opt_stats_opts(const char *v, size_t vlen, char *dest) {
size_t opts_len = strlen(dest);
assert(opts_len <= stats_print_tot_num_options);
for (size_t i = 0; i < vlen; i++) {
......@@ -763,16 +802,16 @@ init_opt_stats_print_opts(const char *v, size_t vlen) {
default: continue;
}
if (strchr(opt_stats_print_opts, v[i]) != NULL) {
if (strchr(dest, v[i]) != NULL) {
/* Ignore repeated. */
continue;
}
opt_stats_print_opts[opts_len++] = v[i];
opt_stats_print_opts[opts_len] = '\0';
dest[opts_len++] = v[i];
dest[opts_len] = '\0';
assert(opts_len <= stats_print_tot_num_options);
}
assert(opts_len == strlen(opt_stats_print_opts));
assert(opts_len == strlen(dest));
}
/* Reads the next size pair in a multi-sized option. */
......@@ -854,10 +893,12 @@ malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
if (opts != *opts_p) {
malloc_write("<jemalloc>: Conf string ends "
"with key\n");
had_conf_error = true;
}
return true;
default:
malloc_write("<jemalloc>: Malformed conf string\n");
had_conf_error = true;
return true;
}
}
......@@ -876,6 +917,7 @@ malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
if (*opts == '\0') {
malloc_write("<jemalloc>: Conf string ends "
"with comma\n");
had_conf_error = true;
}
*vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
accept = true;
......@@ -932,7 +974,7 @@ malloc_slow_flag_init(void) {
}
/* Number of sources for initializing malloc_conf */
#define MALLOC_CONF_NSOURCES 4
#define MALLOC_CONF_NSOURCES 5
static const char *
obtain_malloc_conf(unsigned which_source, char buf[PATH_MAX + 1]) {
......@@ -1010,6 +1052,9 @@ obtain_malloc_conf(unsigned which_source, char buf[PATH_MAX + 1]) {
ret = NULL;
}
break;
} case 4: {
ret = je_malloc_conf_2_conf_harder;
break;
} default:
not_reached();
ret = NULL;
......@@ -1024,9 +1069,11 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
static const char *opts_explain[MALLOC_CONF_NSOURCES] = {
"string specified via --with-malloc-conf",
"string pointed to by the global variable malloc_conf",
"\"name\" of the file referenced by the symbolic link named "
"/etc/malloc.conf",
"value of the environment variable MALLOC_CONF"
("\"name\" of the file referenced by the symbolic link named "
"/etc/malloc.conf"),
"value of the environment variable MALLOC_CONF",
("string pointed to by the global variable "
"malloc_conf_2_conf_harder"),
};
unsigned i;
const char *opts, *k, *v;
......@@ -1094,39 +1141,50 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
#define CONF_CHECK_MIN(um, min) ((um) < (min))
#define CONF_DONT_CHECK_MAX(um, max) false
#define CONF_CHECK_MAX(um, max) ((um) > (max))
#define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \
#define CONF_VALUE_READ(max_t, result) \
char *end; \
set_errno(0); \
result = (max_t)malloc_strtoumax(v, &end, 0);
#define CONF_VALUE_READ_FAIL() \
(get_errno() != 0 || (uintptr_t)end - (uintptr_t)v != vlen)
#define CONF_HANDLE_T(t, max_t, o, n, min, max, check_min, check_max, clip) \
if (CONF_MATCH(n)) { \
uintmax_t um; \
char *end; \
\
set_errno(0); \
um = malloc_strtoumax(v, &end, 0); \
if (get_errno() != 0 || (uintptr_t)end -\
(uintptr_t)v != vlen) { \
max_t mv; \
CONF_VALUE_READ(max_t, mv) \
if (CONF_VALUE_READ_FAIL()) { \
CONF_ERROR("Invalid conf value",\
k, klen, v, vlen); \
} else if (clip) { \
if (check_min(um, (t)(min))) { \
if (check_min(mv, (t)(min))) { \
o = (t)(min); \
} else if ( \
check_max(um, (t)(max))) { \
check_max(mv, (t)(max))) { \
o = (t)(max); \
} else { \
o = (t)um; \
o = (t)mv; \
} \
} else { \
if (check_min(um, (t)(min)) || \
check_max(um, (t)(max))) { \
if (check_min(mv, (t)(min)) || \
check_max(mv, (t)(max))) { \
CONF_ERROR( \
"Out-of-range " \
"conf value", \
k, klen, v, vlen); \
} else { \
o = (t)um; \
o = (t)mv; \
} \
} \
CONF_CONTINUE; \
}
#define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \
CONF_HANDLE_T(t, uintmax_t, o, n, min, max, check_min, \
check_max, clip)
#define CONF_HANDLE_T_SIGNED(t, o, n, min, max, check_min, check_max, clip)\
CONF_HANDLE_T(t, intmax_t, o, n, min, max, check_min, \
check_max, clip)
#define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \
clip) \
CONF_HANDLE_T_U(unsigned, o, n, min, max, \
......@@ -1134,27 +1192,15 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
#define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip) \
CONF_HANDLE_T_U(size_t, o, n, min, max, \
check_min, check_max, clip)
#define CONF_HANDLE_INT64_T(o, n, min, max, check_min, check_max, clip) \
CONF_HANDLE_T_SIGNED(int64_t, o, n, min, max, \
check_min, check_max, clip)
#define CONF_HANDLE_UINT64_T(o, n, min, max, check_min, check_max, clip)\
CONF_HANDLE_T_U(uint64_t, o, n, min, max, \
check_min, check_max, clip)
#define CONF_HANDLE_SSIZE_T(o, n, min, max) \
if (CONF_MATCH(n)) { \
long l; \
char *end; \
\
set_errno(0); \
l = strtol(v, &end, 0); \
if (get_errno() != 0 || (uintptr_t)end -\
(uintptr_t)v != vlen) { \
CONF_ERROR("Invalid conf value",\
k, klen, v, vlen); \
} else if (l < (ssize_t)(min) || l > \
(ssize_t)(max)) { \
CONF_ERROR( \
"Out-of-range conf value", \
k, klen, v, vlen); \
} else { \
o = l; \
} \
CONF_CONTINUE; \
}
CONF_HANDLE_T_SIGNED(ssize_t, o, n, min, max, \
CONF_CHECK_MIN, CONF_CHECK_MAX, false)
#define CONF_HANDLE_CHAR_P(o, n, d) \
if (CONF_MATCH(n)) { \
size_t cpylen = (vlen <= \
......@@ -1174,13 +1220,14 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
CONF_HANDLE_BOOL(opt_abort, "abort")
CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf")
CONF_HANDLE_BOOL(opt_trust_madvise, "trust_madvise")
if (strncmp("metadata_thp", k, klen) == 0) {
int i;
int m;
bool match = false;
for (i = 0; i < metadata_thp_mode_limit; i++) {
if (strncmp(metadata_thp_mode_names[i],
for (m = 0; m < metadata_thp_mode_limit; m++) {
if (strncmp(metadata_thp_mode_names[m],
v, vlen) == 0) {
opt_metadata_thp = i;
opt_metadata_thp = m;
match = true;
break;
}
......@@ -1193,18 +1240,18 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
}
CONF_HANDLE_BOOL(opt_retain, "retain")
if (strncmp("dss", k, klen) == 0) {
int i;
int m;
bool match = false;
for (i = 0; i < dss_prec_limit; i++) {
if (strncmp(dss_prec_names[i], v, vlen)
for (m = 0; m < dss_prec_limit; m++) {
if (strncmp(dss_prec_names[m], v, vlen)
== 0) {
if (extent_dss_prec_set(i)) {
if (extent_dss_prec_set(m)) {
CONF_ERROR(
"Error setting dss",
k, klen, v, vlen);
} else {
opt_dss =
dss_prec_names[i];
dss_prec_names[m];
match = true;
break;
}
......@@ -1216,9 +1263,27 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
}
CONF_CONTINUE;
}
CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
UINT_MAX, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX,
false)
if (CONF_MATCH("narenas")) {
if (CONF_MATCH_VALUE("default")) {
opt_narenas = 0;
CONF_CONTINUE;
} else {
CONF_HANDLE_UNSIGNED(opt_narenas,
"narenas", 1, UINT_MAX,
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX,
/* clip */ false)
}
}
if (CONF_MATCH("narenas_ratio")) {
char *end;
bool err = fxp_parse(&opt_narenas_ratio, v,
&end);
if (err || (size_t)(end - v) != vlen) {
CONF_ERROR("Invalid conf value",
k, klen, v, vlen);
}
CONF_CONTINUE;
}
if (CONF_MATCH("bin_shards")) {
const char *bin_shards_segment_cur = v;
size_t vlen_left = vlen;
......@@ -1241,6 +1306,9 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
} while (vlen_left > 0);
CONF_CONTINUE;
}
CONF_HANDLE_INT64_T(opt_mutex_max_spin,
"mutex_max_spin", -1, INT64_MAX, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, false);
CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms,
"dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
......@@ -1251,7 +1319,16 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
SSIZE_MAX);
CONF_HANDLE_BOOL(opt_stats_print, "stats_print")
if (CONF_MATCH("stats_print_opts")) {
init_opt_stats_print_opts(v, vlen);
init_opt_stats_opts(v, vlen,
opt_stats_print_opts);
CONF_CONTINUE;
}
CONF_HANDLE_INT64_T(opt_stats_interval,
"stats_interval", -1, INT64_MAX,
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, false)
if (CONF_MATCH("stats_interval_opts")) {
init_opt_stats_opts(v, vlen,
opt_stats_interval_opts);
CONF_CONTINUE;
}
if (config_fill) {
......@@ -1287,9 +1364,61 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
if (config_xmalloc) {
CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
}
if (config_enable_cxx) {
CONF_HANDLE_BOOL(
opt_experimental_infallible_new,
"experimental_infallible_new")
}
CONF_HANDLE_BOOL(opt_tcache, "tcache")
CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max",
-1, (sizeof(size_t) << 3) - 1)
CONF_HANDLE_SIZE_T(opt_tcache_max, "tcache_max",
0, TCACHE_MAXCLASS_LIMIT, CONF_DONT_CHECK_MIN,
CONF_CHECK_MAX, /* clip */ true)
if (CONF_MATCH("lg_tcache_max")) {
size_t m;
CONF_VALUE_READ(size_t, m)
if (CONF_VALUE_READ_FAIL()) {
CONF_ERROR("Invalid conf value",
k, klen, v, vlen);
} else {
/* clip if necessary */
if (m > TCACHE_LG_MAXCLASS_LIMIT) {
m = TCACHE_LG_MAXCLASS_LIMIT;
}
opt_tcache_max = (size_t)1 << m;
}
CONF_CONTINUE;
}
/*
* Anyone trying to set a value outside -16 to 16 is
* deeply confused.
*/
CONF_HANDLE_SSIZE_T(opt_lg_tcache_nslots_mul,
"lg_tcache_nslots_mul", -16, 16)
/* Ditto with values past 2048. */
CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_min,
"tcache_nslots_small_min", 1, 2048,
CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_max,
"tcache_nslots_small_max", 1, 2048,
CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
CONF_HANDLE_UNSIGNED(opt_tcache_nslots_large,
"tcache_nslots_large", 1, 2048,
CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
CONF_HANDLE_SIZE_T(opt_tcache_gc_incr_bytes,
"tcache_gc_incr_bytes", 1024, SIZE_T_MAX,
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX,
/* clip */ true)
CONF_HANDLE_SIZE_T(opt_tcache_gc_delay_bytes,
"tcache_gc_delay_bytes", 0, SIZE_T_MAX,
CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX,
/* clip */ false)
CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_small_div,
"lg_tcache_flush_small_div", 1, 16,
CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_large_div,
"lg_tcache_flush_large_div", 1, 16,
CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
/*
* The runtime option of oversize_threshold remains
......@@ -1309,16 +1438,16 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
if (strncmp("percpu_arena", k, klen) == 0) {
bool match = false;
for (int i = percpu_arena_mode_names_base; i <
percpu_arena_mode_names_limit; i++) {
if (strncmp(percpu_arena_mode_names[i],
for (int m = percpu_arena_mode_names_base; m <
percpu_arena_mode_names_limit; m++) {
if (strncmp(percpu_arena_mode_names[m],
v, vlen) == 0) {
if (!have_percpu_arena) {
CONF_ERROR(
"No getcpu support",
k, klen, v, vlen);
}
opt_percpu_arena = i;
opt_percpu_arena = m;
match = true;
break;
}
......@@ -1336,7 +1465,83 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
opt_max_background_threads,
CONF_CHECK_MIN, CONF_CHECK_MAX,
true);
CONF_HANDLE_BOOL(opt_hpa, "hpa")
CONF_HANDLE_SIZE_T(opt_hpa_opts.slab_max_alloc,
"hpa_slab_max_alloc", PAGE, HUGEPAGE,
CONF_CHECK_MIN, CONF_CHECK_MAX, true);
/*
* Accept either a ratio-based or an exact hugification
* threshold.
*/
CONF_HANDLE_SIZE_T(opt_hpa_opts.hugification_threshold,
"hpa_hugification_threshold", PAGE, HUGEPAGE,
CONF_CHECK_MIN, CONF_CHECK_MAX, true);
if (CONF_MATCH("hpa_hugification_threshold_ratio")) {
fxp_t ratio;
char *end;
bool err = fxp_parse(&ratio, v,
&end);
if (err || (size_t)(end - v) != vlen
|| ratio > FXP_INIT_INT(1)) {
CONF_ERROR("Invalid conf value",
k, klen, v, vlen);
} else {
opt_hpa_opts.hugification_threshold =
fxp_mul_frac(HUGEPAGE, ratio);
}
CONF_CONTINUE;
}
CONF_HANDLE_UINT64_T(
opt_hpa_opts.hugify_delay_ms, "hpa_hugify_delay_ms",
0, 0, CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX,
false);
CONF_HANDLE_UINT64_T(
opt_hpa_opts.min_purge_interval_ms,
"hpa_min_purge_interval_ms", 0, 0,
CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false);
if (CONF_MATCH("hpa_dirty_mult")) {
if (CONF_MATCH_VALUE("-1")) {
opt_hpa_opts.dirty_mult = (fxp_t)-1;
CONF_CONTINUE;
}
fxp_t ratio;
char *end;
bool err = fxp_parse(&ratio, v,
&end);
if (err || (size_t)(end - v) != vlen) {
CONF_ERROR("Invalid conf value",
k, klen, v, vlen);
} else {
opt_hpa_opts.dirty_mult = ratio;
}
CONF_CONTINUE;
}
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards,
"hpa_sec_nshards", 0, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_alloc,
"hpa_sec_max_alloc", PAGE, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
"hpa_sec_max_bytes", PAGE, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.bytes_after_flush,
"hpa_sec_bytes_after_flush", PAGE, 0,
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra,
"hpa_sec_batch_fill_extra", 0, HUGEPAGE_PAGES,
CONF_CHECK_MIN, CONF_CHECK_MAX, true);
if (CONF_MATCH("slab_sizes")) {
if (CONF_MATCH_VALUE("default")) {
sc_data_init(sc_data);
CONF_CONTINUE;
}
bool err;
const char *slab_size_segment_cur = v;
size_t vlen_left = vlen;
......@@ -1378,7 +1583,44 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump")
CONF_HANDLE_BOOL(opt_prof_final, "prof_final")
CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak")
CONF_HANDLE_BOOL(opt_prof_leak_error,
"prof_leak_error")
CONF_HANDLE_BOOL(opt_prof_log, "prof_log")
CONF_HANDLE_SSIZE_T(opt_prof_recent_alloc_max,
"prof_recent_alloc_max", -1, SSIZE_MAX)
CONF_HANDLE_BOOL(opt_prof_stats, "prof_stats")
CONF_HANDLE_BOOL(opt_prof_sys_thread_name,
"prof_sys_thread_name")
if (CONF_MATCH("prof_time_resolution")) {
if (CONF_MATCH_VALUE("default")) {
opt_prof_time_res =
prof_time_res_default;
} else if (CONF_MATCH_VALUE("high")) {
if (!config_high_res_timer) {
CONF_ERROR(
"No high resolution"
" timer support",
k, klen, v, vlen);
} else {
opt_prof_time_res =
prof_time_res_high;
}
} else {
CONF_ERROR("Invalid conf value",
k, klen, v, vlen);
}
CONF_CONTINUE;
}
/*
* Undocumented. When set to false, don't
* correct for an unbiasing bug in jeprof
* attribution. This can be handy if you want
* to get consistent numbers from your binary
* across different jemalloc versions, even if
* those numbers are incorrect. The default is
* true.
*/
CONF_HANDLE_BOOL(opt_prof_unbias, "prof_unbias")
}
if (config_log) {
if (CONF_MATCH("log")) {
......@@ -1392,15 +1634,15 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
}
if (CONF_MATCH("thp")) {
bool match = false;
for (int i = 0; i < thp_mode_names_limit; i++) {
if (strncmp(thp_mode_names[i],v, vlen)
for (int m = 0; m < thp_mode_names_limit; m++) {
if (strncmp(thp_mode_names[m],v, vlen)
== 0) {
if (!have_madvise_huge) {
if (!have_madvise_huge && !have_memcntl) {
CONF_ERROR(
"No THP support",
k, klen, v, vlen);
}
opt_thp = i;
opt_thp = m;
match = true;
break;
}
......@@ -1411,6 +1653,55 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
}
CONF_CONTINUE;
}
if (CONF_MATCH("zero_realloc")) {
if (CONF_MATCH_VALUE("alloc")) {
opt_zero_realloc_action
= zero_realloc_action_alloc;
} else if (CONF_MATCH_VALUE("free")) {
opt_zero_realloc_action
= zero_realloc_action_free;
} else if (CONF_MATCH_VALUE("abort")) {
opt_zero_realloc_action
= zero_realloc_action_abort;
} else {
CONF_ERROR("Invalid conf value",
k, klen, v, vlen);
}
CONF_CONTINUE;
}
if (config_uaf_detection &&
CONF_MATCH("lg_san_uaf_align")) {
ssize_t a;
CONF_VALUE_READ(ssize_t, a)
if (CONF_VALUE_READ_FAIL() || a < -1) {
CONF_ERROR("Invalid conf value",
k, klen, v, vlen);
}
if (a == -1) {
opt_lg_san_uaf_align = -1;
CONF_CONTINUE;
}
/* clip if necessary */
ssize_t max_allowed = (sizeof(size_t) << 3) - 1;
ssize_t min_allowed = LG_PAGE;
if (a > max_allowed) {
a = max_allowed;
} else if (a < min_allowed) {
a = min_allowed;
}
opt_lg_san_uaf_align = a;
CONF_CONTINUE;
}
CONF_HANDLE_SIZE_T(opt_san_guard_small,
"san_guard_small", 0, SIZE_T_MAX,
CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false)
CONF_HANDLE_SIZE_T(opt_san_guard_large,
"san_guard_large", 0, SIZE_T_MAX,
CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false)
CONF_ERROR("Invalid conf pair", k, klen, v, vlen);
#undef CONF_ERROR
#undef CONF_CONTINUE
......@@ -1421,7 +1712,9 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
#undef CONF_CHECK_MIN
#undef CONF_DONT_CHECK_MAX
#undef CONF_CHECK_MAX
#undef CONF_HANDLE_T
#undef CONF_HANDLE_T_U
#undef CONF_HANDLE_T_SIGNED
#undef CONF_HANDLE_UNSIGNED
#undef CONF_HANDLE_SIZE_T
#undef CONF_HANDLE_SSIZE_T
......@@ -1436,15 +1729,33 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
atomic_store_b(&log_init_done, true, ATOMIC_RELEASE);
}
static bool
malloc_conf_init_check_deps(void) {
if (opt_prof_leak_error && !opt_prof_final) {
malloc_printf("<jemalloc>: prof_leak_error is set w/o "
"prof_final.\n");
return true;
}
return false;
}
static void
malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL};
const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL,
NULL};
char buf[PATH_MAX + 1];
/* The first call only set the confirm_conf option and opts_cache */
malloc_conf_init_helper(NULL, NULL, true, opts_cache, buf);
malloc_conf_init_helper(sc_data, bin_shard_sizes, false, opts_cache,
NULL);
if (malloc_conf_init_check_deps()) {
/* check_deps does warning msg only; abort below if needed. */
if (opt_abort_conf) {
malloc_abort_invalid_conf();
}
}
}
#undef MALLOC_CONF_NSOURCES
......@@ -1488,8 +1799,8 @@ malloc_init_hard_a0_locked() {
* Ordering here is somewhat tricky; we need sc_boot() first, since that
* determines what the size classes will be, and then
* malloc_conf_init(), since any slab size tweaking will need to be done
* before sz_boot and bin_boot, which assume that the values they read
* out of sc_data_global are final.
* before sz_boot and bin_info_boot, which assume that the values they
* read out of sc_data_global are final.
*/
sc_boot(&sc_data);
unsigned bin_shard_sizes[SC_NBINS];
......@@ -1503,8 +1814,9 @@ malloc_init_hard_a0_locked() {
prof_boot0();
}
malloc_conf_init(&sc_data, bin_shard_sizes);
sz_boot(&sc_data);
bin_boot(&sc_data, bin_shard_sizes);
san_init(opt_lg_san_uaf_align);
sz_boot(&sc_data, opt_cache_oblivious);
bin_info_boot(&sc_data, bin_shard_sizes);
if (opt_stats_print) {
/* Print statistics at exit. */
......@@ -1515,12 +1827,20 @@ malloc_init_hard_a0_locked() {
}
}
}
if (stats_boot()) {
return true;
}
if (pages_boot()) {
return true;
}
if (base_boot(TSDN_NULL)) {
return true;
}
/* emap_global is static, hence zeroed. */
if (emap_init(&arena_emap_global, b0get(), /* zeroed */ true)) {
return true;
}
if (extent_boot()) {
return true;
}
......@@ -1530,8 +1850,20 @@ malloc_init_hard_a0_locked() {
if (config_prof) {
prof_boot1();
}
arena_boot(&sc_data);
if (tcache_boot(TSDN_NULL)) {
if (opt_hpa && !hpa_supported()) {
malloc_printf("<jemalloc>: HPA not supported in the current "
"configuration; %s.",
opt_abort_conf ? "aborting" : "disabling");
if (opt_abort_conf) {
malloc_abort_invalid_conf();
} else {
opt_hpa = false;
}
}
if (arena_boot(&sc_data, b0get(), opt_hpa)) {
return true;
}
if (tcache_boot(TSDN_NULL, b0get())) {
return true;
}
if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS,
......@@ -1550,11 +1882,29 @@ malloc_init_hard_a0_locked() {
* Initialize one arena here. The rest are lazily created in
* arena_choose_hard().
*/
if (arena_init(TSDN_NULL, 0, (extent_hooks_t *)&extent_hooks_default)
== NULL) {
if (arena_init(TSDN_NULL, 0, &arena_config_default) == NULL) {
return true;
}
a0 = arena_get(TSDN_NULL, 0, false);
if (opt_hpa && !hpa_supported()) {
malloc_printf("<jemalloc>: HPA not supported in the current "
"configuration; %s.",
opt_abort_conf ? "aborting" : "disabling");
if (opt_abort_conf) {
malloc_abort_invalid_conf();
} else {
opt_hpa = false;
}
} else if (opt_hpa) {
hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts;
hpa_shard_opts.deferral_allowed = background_thread_enabled();
if (pa_shard_enable_hpa(TSDN_NULL, &a0->pa_shard,
&hpa_shard_opts, &opt_hpa_sec_opts)) {
return true;
}
}
malloc_init_state = malloc_init_a0_initialized;
return false;
......@@ -1576,6 +1926,29 @@ malloc_init_hard_recursible(void) {
malloc_init_state = malloc_init_recursible;
ncpus = malloc_ncpus();
if (opt_percpu_arena != percpu_arena_disabled) {
bool cpu_count_is_deterministic =
malloc_cpu_count_is_deterministic();
if (!cpu_count_is_deterministic) {
/*
* If # of CPU is not deterministic, and narenas not
* specified, disables per cpu arena since it may not
* detect CPU IDs properly.
*/
if (opt_narenas == 0) {
opt_percpu_arena = percpu_arena_disabled;
malloc_write("<jemalloc>: Number of CPUs "
"detected is not deterministic. Per-CPU "
"arena disabled.\n");
if (opt_abort_conf) {
malloc_abort_invalid_conf();
}
if (opt_abort) {
abort();
}
}
}
}
#if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
&& !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \
......@@ -1606,7 +1979,13 @@ malloc_narenas_default(void) {
* default.
*/
if (ncpus > 1) {
return ncpus << 2;
fxp_t fxp_ncpus = FXP_INIT_INT(ncpus);
fxp_t goal = fxp_mul(fxp_ncpus, opt_narenas_ratio);
uint32_t int_goal = fxp_round_nearest(goal);
if (int_goal == 0) {
return 1;
}
return int_goal;
} else {
return 1;
}
......@@ -1765,10 +2144,11 @@ malloc_init_hard(void) {
/* Set reentrancy level to 1 during init. */
pre_reentrancy(tsd, NULL);
/* Initialize narenas before prof_boot2 (for allocation). */
if (malloc_init_narenas() || background_thread_boot1(tsd_tsdn(tsd))) {
if (malloc_init_narenas()
|| background_thread_boot1(tsd_tsdn(tsd), b0get())) {
UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
}
if (config_prof && prof_boot2(tsd)) {
if (config_prof && prof_boot2(tsd, b0get())) {
UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
}
......@@ -1907,38 +2287,107 @@ dynamic_opts_init(dynamic_opts_t *dynamic_opts) {
dynamic_opts->arena_ind = ARENA_IND_AUTOMATIC;
}
/* ind is ignored if dopts->alignment > 0. */
JEMALLOC_ALWAYS_INLINE void *
imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
size_t size, size_t usize, szind_t ind) {
tcache_t *tcache;
arena_t *arena;
/*
* ind parameter is optional and is only checked and filled if alignment == 0;
* return true if result is out of range.
*/
JEMALLOC_ALWAYS_INLINE bool
aligned_usize_get(size_t size, size_t alignment, size_t *usize, szind_t *ind,
bool bump_empty_aligned_alloc) {
assert(usize != NULL);
if (alignment == 0) {
if (ind != NULL) {
*ind = sz_size2index(size);
if (unlikely(*ind >= SC_NSIZES)) {
return true;
}
*usize = sz_index2size(*ind);
assert(*usize > 0 && *usize <= SC_LARGE_MAXCLASS);
return false;
}
*usize = sz_s2u(size);
} else {
if (bump_empty_aligned_alloc && unlikely(size == 0)) {
size = 1;
}
*usize = sz_sa2u(size, alignment);
}
if (unlikely(*usize == 0 || *usize > SC_LARGE_MAXCLASS)) {
return true;
}
return false;
}
/* Fill in the tcache. */
if (dopts->tcache_ind == TCACHE_IND_AUTOMATIC) {
if (likely(!sopts->slow)) {
JEMALLOC_ALWAYS_INLINE bool
zero_get(bool guarantee, bool slow) {
if (config_fill && slow && unlikely(opt_zero)) {
return true;
} else {
return guarantee;
}
}
JEMALLOC_ALWAYS_INLINE tcache_t *
tcache_get_from_ind(tsd_t *tsd, unsigned tcache_ind, bool slow, bool is_alloc) {
tcache_t *tcache;
if (tcache_ind == TCACHE_IND_AUTOMATIC) {
if (likely(!slow)) {
/* Getting tcache ptr unconditionally. */
tcache = tsd_tcachep_get(tsd);
assert(tcache == tcache_get(tsd));
} else {
} else if (is_alloc ||
likely(tsd_reentrancy_level_get(tsd) == 0)) {
tcache = tcache_get(tsd);
} else {
tcache = NULL;
}
} else if (dopts->tcache_ind == TCACHE_IND_NONE) {
tcache = NULL;
} else {
tcache = tcaches_get(tsd, dopts->tcache_ind);
/*
* Should not specify tcache on deallocation path when being
* reentrant.
*/
assert(is_alloc || tsd_reentrancy_level_get(tsd) == 0 ||
tsd_state_nocleanup(tsd));
if (tcache_ind == TCACHE_IND_NONE) {
tcache = NULL;
} else {
tcache = tcaches_get(tsd, tcache_ind);
}
}
return tcache;
}
/* Fill in the arena. */
if (dopts->arena_ind == ARENA_IND_AUTOMATIC) {
/* Return true if a manual arena is specified and arena_get() OOMs. */
JEMALLOC_ALWAYS_INLINE bool
arena_get_from_ind(tsd_t *tsd, unsigned arena_ind, arena_t **arena_p) {
if (arena_ind == ARENA_IND_AUTOMATIC) {
/*
* In case of automatic arena management, we defer arena
* computation until as late as we can, hoping to fill the
* allocation out of the tcache.
*/
arena = NULL;
*arena_p = NULL;
} else {
arena = arena_get(tsd_tsdn(tsd), dopts->arena_ind, true);
*arena_p = arena_get(tsd_tsdn(tsd), arena_ind, true);
if (unlikely(*arena_p == NULL) && arena_ind >= narenas_auto) {
return true;
}
}
return false;
}
/* ind is ignored if dopts->alignment > 0. */
JEMALLOC_ALWAYS_INLINE void *
imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
size_t size, size_t usize, szind_t ind) {
/* Fill in the tcache. */
tcache_t *tcache = tcache_get_from_ind(tsd, dopts->tcache_ind,
sopts->slow, /* is_alloc */ true);
/* Fill in the arena. */
arena_t *arena;
if (arena_get_from_ind(tsd, dopts->arena_ind, &arena)) {
return NULL;
}
if (unlikely(dopts->alignment != 0)) {
......@@ -1962,6 +2411,7 @@ imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
szind_t ind_large;
size_t bumped_usize = usize;
dopts->alignment = prof_sample_align(dopts->alignment);
if (usize <= SC_SMALL_MAXCLASS) {
assert(((dopts->alignment == 0) ?
sz_s2u(SC_LARGE_MINCLASS) :
......@@ -1978,6 +2428,7 @@ imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
} else {
ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind);
}
assert(prof_sample_aligned(ret));
return ret;
}
......@@ -2031,16 +2482,14 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
/* Filled in by compute_size_with_overflow below. */
size_t size = 0;
/*
* For unaligned allocations, we need only ind. For aligned
* allocations, or in case of stats or profiling we need usize.
*
* These are actually dead stores, in that their values are reset before
* any branch on their value is taken. Sometimes though, it's
* convenient to pass them as arguments before this point. To avoid
* undefined behavior then, we initialize them with dummy stores.
* The zero initialization for ind is actually dead store, in that its
* value is reset before any branch on its value is taken. Sometimes
* though, it's convenient to pass it as arguments before this point.
* To avoid undefined behavior then, we initialize it with dummy stores.
*/
szind_t ind = 0;
size_t usize = 0;
/* usize will always be properly initialized. */
size_t usize;
/* Reentrancy is only checked on slow path. */
int8_t reentrancy_level;
......@@ -2057,31 +2506,12 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
}
/* This is the beginning of the "core" algorithm. */
if (dopts->alignment == 0) {
ind = sz_size2index(size);
if (unlikely(ind >= SC_NSIZES)) {
goto label_oom;
}
if (config_stats || (config_prof && opt_prof) || sopts->usize) {
usize = sz_index2size(ind);
dopts->usize = usize;
assert(usize > 0 && usize
<= SC_LARGE_MAXCLASS);
}
} else {
if (sopts->bump_empty_aligned_alloc) {
if (unlikely(size == 0)) {
size = 1;
}
}
usize = sz_sa2u(size, dopts->alignment);
dopts->usize = usize;
if (unlikely(usize == 0
|| usize > SC_LARGE_MAXCLASS)) {
goto label_oom;
}
dopts->zero = zero_get(dopts->zero, sopts->slow);
if (aligned_usize_get(size, dopts->alignment, &usize, &ind,
sopts->bump_empty_aligned_alloc)) {
goto label_oom;
}
dopts->usize = usize;
/* Validate the user input. */
if (sopts->assert_nonempty_alloc) {
assert (size != 0);
......@@ -2107,26 +2537,25 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
dopts->arena_ind = 0;
}
/*
* If dopts->alignment > 0, then ind is still 0, but usize was computed
* in the previous if statement. Down the positive alignment path,
* imalloc_no_sample and imalloc_sample will ignore ind.
*/
/* If profiling is on, get our profiling context. */
if (config_prof && opt_prof) {
/*
* Note that if we're going down this path, usize must have been
* initialized in the previous if statement.
*/
prof_tctx_t *tctx = prof_alloc_prep(
tsd, usize, prof_active_get_unlocked(), true);
bool prof_active = prof_active_get_unlocked();
bool sample_event = te_prof_sample_event_lookahead(tsd, usize);
prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active,
sample_event);
alloc_ctx_t alloc_ctx;
emap_alloc_ctx_t alloc_ctx;
if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
alloc_ctx.slab = (usize
<= SC_SMALL_MAXCLASS);
alloc_ctx.slab = (usize <= SC_SMALL_MAXCLASS);
allocation = imalloc_no_sample(
sopts, dopts, tsd, usize, usize, ind);
} else if ((uintptr_t)tctx > (uintptr_t)1U) {
/*
* Note that ind might still be 0 here. This is fine;
* imalloc_sample ignores ind if dopts->alignment > 0.
*/
allocation = imalloc_sample(
sopts, dopts, tsd, usize, ind);
alloc_ctx.slab = false;
......@@ -2135,17 +2564,12 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
}
if (unlikely(allocation == NULL)) {
prof_alloc_rollback(tsd, tctx, true);
prof_alloc_rollback(tsd, tctx);
goto label_oom;
}
prof_malloc(tsd_tsdn(tsd), allocation, usize, &alloc_ctx, tctx);
prof_malloc(tsd, allocation, size, usize, &alloc_ctx, tctx);
} else {
/*
* If dopts->alignment > 0, then ind is still 0, but usize was
* computed in the previous if statement. Down the positive
* alignment path, imalloc_no_sample ignores ind and size
* (relying only on usize).
*/
assert(!opt_prof);
allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
ind);
if (unlikely(allocation == NULL)) {
......@@ -2157,12 +2581,17 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
* Allocation has been done at this point. We still have some
* post-allocation work to do though.
*/
thread_alloc_event(tsd, usize);
assert(dopts->alignment == 0
|| ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0));
if (config_stats) {
assert(usize == isalloc(tsd_tsdn(tsd), allocation));
*tsd_thread_allocatedp_get(tsd) += usize;
assert(usize == isalloc(tsd_tsdn(tsd), allocation));
if (config_fill && sopts->slow && !dopts->zero
&& unlikely(opt_junk_alloc)) {
junk_alloc_callback(allocation, usize);
}
if (sopts->slow) {
......@@ -2273,7 +2702,11 @@ malloc_default(size_t size) {
static_opts_t sopts;
dynamic_opts_t dopts;
LOG("core.malloc.entry", "size: %zu", size);
/*
* This variant has logging hook on exit but not on entry. It's callled
* only by je_malloc, below, which emits the entry one for us (and, if
* it calls us, does so only via tail call).
*/
static_opts_init(&sopts);
dynamic_opts_init(&dopts);
......@@ -2306,86 +2739,11 @@ malloc_default(size_t size) {
* Begin malloc(3)-compatible functions.
*/
/*
* malloc() fastpath.
*
* Fastpath assumes size <= SC_LOOKUP_MAXCLASS, and that we hit
* tcache. If either of these is false, we tail-call to the slowpath,
* malloc_default(). Tail-calling is used to avoid any caller-saved
* registers.
*
* fastpath supports ticker and profiling, both of which will also
* tail-call to the slowpath if they fire.
*/
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
je_malloc(size_t size) {
LOG("core.malloc.entry", "size: %zu", size);
if (tsd_get_allocates() && unlikely(!malloc_initialized())) {
return malloc_default(size);
}
tsd_t *tsd = tsd_get(false);
if (unlikely(!tsd || !tsd_fast(tsd) || (size > SC_LOOKUP_MAXCLASS))) {
return malloc_default(size);
}
tcache_t *tcache = tsd_tcachep_get(tsd);
if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
return malloc_default(size);
}
szind_t ind = sz_size2index_lookup(size);
size_t usize;
if (config_stats || config_prof) {
usize = sz_index2size(ind);
}
/* Fast path relies on size being a bin. I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS */
assert(ind < SC_NBINS);
assert(size <= SC_SMALL_MAXCLASS);
if (config_prof) {
int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd);
bytes_until_sample -= usize;
tsd_bytes_until_sample_set(tsd, bytes_until_sample);
if (unlikely(bytes_until_sample < 0)) {
/*
* Avoid a prof_active check on the fastpath.
* If prof_active is false, set bytes_until_sample to
* a large value. If prof_active is set to true,
* bytes_until_sample will be reset.
*/
if (!prof_active) {
tsd_bytes_until_sample_set(tsd, SSIZE_MAX);
}
return malloc_default(size);
}
}
cache_bin_t *bin = tcache_small_bin_get(tcache, ind);
bool tcache_success;
void* ret = cache_bin_alloc_easy(bin, &tcache_success);
if (tcache_success) {
if (config_stats) {
*tsd_thread_allocatedp_get(tsd) += usize;
bin->tstats.nrequests++;
}
if (config_prof) {
tcache->prof_accumbytes += usize;
}
LOG("core.malloc.exit", "result: %p", ret);
/* Fastpath success */
return ret;
}
return malloc_default(size);
return imalloc_fastpath(size, &malloc_default);
}
JEMALLOC_EXPORT int JEMALLOC_NOTHROW
......@@ -2502,56 +2860,6 @@ je_calloc(size_t num, size_t size) {
return ret;
}
static void *
irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) {
void *p;
if (tctx == NULL) {
return NULL;
}
if (usize <= SC_SMALL_MAXCLASS) {
p = iralloc(tsd, old_ptr, old_usize,
SC_LARGE_MINCLASS, 0, false, hook_args);
if (p == NULL) {
return NULL;
}
arena_prof_promote(tsd_tsdn(tsd), p, usize);
} else {
p = iralloc(tsd, old_ptr, old_usize, usize, 0, false,
hook_args);
}
return p;
}
JEMALLOC_ALWAYS_INLINE void *
irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) {
void *p;
bool prof_active;
prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked();
old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx);
tctx = prof_alloc_prep(tsd, usize, prof_active, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx,
hook_args);
} else {
p = iralloc(tsd, old_ptr, old_usize, usize, 0, false,
hook_args);
}
if (unlikely(p == NULL)) {
prof_alloc_rollback(tsd, tctx, true);
return NULL;
}
prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
old_tctx);
return p;
}
JEMALLOC_ALWAYS_INLINE void
ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
if (!slow_path) {
......@@ -2565,30 +2873,50 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
emap_alloc_ctx_t alloc_ctx;
emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
&alloc_ctx);
assert(alloc_ctx.szind != SC_NSIZES);
size_t usize;
size_t usize = sz_index2size(alloc_ctx.szind);
if (config_prof && opt_prof) {
usize = sz_index2size(alloc_ctx.szind);
prof_free(tsd, ptr, usize, &alloc_ctx);
} else if (config_stats) {
usize = sz_index2size(alloc_ctx.szind);
}
if (config_stats) {
*tsd_thread_deallocatedp_get(tsd) += usize;
}
if (likely(!slow_path)) {
idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
false);
} else {
if (config_fill && slow_path && opt_junk_free) {
junk_free_callback(ptr, usize);
}
idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
true);
}
thread_dalloc_event(tsd, usize);
}
JEMALLOC_ALWAYS_INLINE bool
maybe_check_alloc_ctx(tsd_t *tsd, void *ptr, emap_alloc_ctx_t *alloc_ctx) {
if (config_opt_size_checks) {
emap_alloc_ctx_t dbg_ctx;
emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
&dbg_ctx);
if (alloc_ctx->szind != dbg_ctx.szind) {
safety_check_fail_sized_dealloc(
/* current_dealloc */ true, ptr,
/* true_size */ sz_size2index(dbg_ctx.szind),
/* input_size */ sz_size2index(alloc_ctx->szind));
return true;
}
if (alloc_ctx->slab != dbg_ctx.slab) {
safety_check_fail(
"Internal heap corruption detected: "
"mismatch in slab bit");
return true;
}
}
return false;
}
JEMALLOC_ALWAYS_INLINE void
......@@ -2604,166 +2932,63 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
alloc_ctx_t alloc_ctx, *ctx;
if (!config_cache_oblivious && ((uintptr_t)ptr & PAGE_MASK) != 0) {
/*
* When cache_oblivious is disabled and ptr is not page aligned,
* the allocation was not sampled -- usize can be used to
* determine szind directly.
*/
emap_alloc_ctx_t alloc_ctx;
if (!config_prof) {
alloc_ctx.szind = sz_size2index(usize);
alloc_ctx.slab = true;
ctx = &alloc_ctx;
if (config_debug) {
alloc_ctx_t dbg_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree,
rtree_ctx, (uintptr_t)ptr, true, &dbg_ctx.szind,
&dbg_ctx.slab);
assert(dbg_ctx.szind == alloc_ctx.szind);
assert(dbg_ctx.slab == alloc_ctx.slab);
}
} else if (config_prof && opt_prof) {
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
assert(alloc_ctx.szind == sz_size2index(usize));
ctx = &alloc_ctx;
alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
} else {
ctx = NULL;
if (likely(!prof_sample_aligned(ptr))) {
/*
* When the ptr is not page aligned, it was not sampled.
* usize can be trusted to determine szind and slab.
*/
alloc_ctx.szind = sz_size2index(usize);
alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
} else if (opt_prof) {
emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global,
ptr, &alloc_ctx);
if (config_opt_safety_checks) {
/* Small alloc may have !slab (sampled). */
if (unlikely(alloc_ctx.szind !=
sz_size2index(usize))) {
safety_check_fail_sized_dealloc(
/* current_dealloc */ true, ptr,
/* true_size */ sz_index2size(
alloc_ctx.szind),
/* input_size */ usize);
}
}
} else {
alloc_ctx.szind = sz_size2index(usize);
alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
}
}
bool fail = maybe_check_alloc_ctx(tsd, ptr, &alloc_ctx);
if (fail) {
/*
* This is a heap corruption bug. In real life we'll crash; for
* the unit test we just want to avoid breaking anything too
* badly to get a test result out. Let's leak instead of trying
* to free.
*/
return;
}
if (config_prof && opt_prof) {
prof_free(tsd, ptr, usize, ctx);
}
if (config_stats) {
*tsd_thread_deallocatedp_get(tsd) += usize;
prof_free(tsd, ptr, usize, &alloc_ctx);
}
if (likely(!slow_path)) {
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false);
} else {
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true);
}
}
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ALLOC_SIZE(2)
je_realloc(void *ptr, size_t arg_size) {
void *ret;
tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL);
size_t usize JEMALLOC_CC_SILENCE_INIT(0);
size_t old_usize = 0;
size_t size = arg_size;
LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);
if (unlikely(size == 0)) {
if (ptr != NULL) {
/* realloc(ptr, 0) is equivalent to free(ptr). */
UTRACE(ptr, 0, 0);
tcache_t *tcache;
tsd_t *tsd = tsd_fetch();
if (tsd_reentrancy_level_get(tsd) == 0) {
tcache = tcache_get(tsd);
} else {
tcache = NULL;
}
uintptr_t args[3] = {(uintptr_t)ptr, size};
hook_invoke_dalloc(hook_dalloc_realloc, ptr, args);
ifree(tsd, ptr, tcache, true);
LOG("core.realloc.exit", "result: %p", NULL);
return NULL;
}
size = 1;
}
if (likely(ptr != NULL)) {
assert(malloc_initialized() || IS_INITIALIZER);
tsd_t *tsd = tsd_fetch();
check_entry_exit_locking(tsd_tsdn(tsd));
hook_ralloc_args_t hook_args = {true, {(uintptr_t)ptr,
(uintptr_t)arg_size, 0, 0}};
alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES);
old_usize = sz_index2size(alloc_ctx.szind);
assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
if (config_prof && opt_prof) {
usize = sz_s2u(size);
if (unlikely(usize == 0
|| usize > SC_LARGE_MAXCLASS)) {
ret = NULL;
} else {
ret = irealloc_prof(tsd, ptr, old_usize, usize,
&alloc_ctx, &hook_args);
}
} else {
if (config_stats) {
usize = sz_s2u(size);
}
ret = iralloc(tsd, ptr, old_usize, size, 0, false,
&hook_args);
}
tsdn = tsd_tsdn(tsd);
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, &alloc_ctx,
false);
} else {
/* realloc(NULL, size) is equivalent to malloc(size). */
static_opts_t sopts;
dynamic_opts_t dopts;
static_opts_init(&sopts);
dynamic_opts_init(&dopts);
sopts.null_out_result_on_error = true;
sopts.set_errno_on_error = true;
sopts.oom_string =
"<jemalloc>: Error in realloc(): out of memory\n";
dopts.result = &ret;
dopts.num_items = 1;
dopts.item_size = size;
imalloc(&sopts, &dopts);
if (sopts.slow) {
uintptr_t args[3] = {(uintptr_t)ptr, arg_size};
hook_invoke_alloc(hook_alloc_realloc, ret,
(uintptr_t)ret, args);
if (config_fill && slow_path && opt_junk_free) {
junk_free_callback(ptr, usize);
}
return ret;
}
if (unlikely(ret == NULL)) {
if (config_xmalloc && unlikely(opt_xmalloc)) {
malloc_write("<jemalloc>: Error in realloc(): "
"out of memory\n");
abort();
}
set_errno(ENOMEM);
}
if (config_stats && likely(ret != NULL)) {
tsd_t *tsd;
assert(usize == isalloc(tsdn, ret));
tsd = tsdn_tsd(tsdn);
*tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize;
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, &alloc_ctx,
true);
}
UTRACE(ptr, size, ret);
check_entry_exit_locking(tsdn);
LOG("core.realloc.exit", "result: %p", ret);
return ret;
thread_dalloc_event(tsd, usize);
}
JEMALLOC_NOINLINE
......@@ -2782,79 +3007,149 @@ free_default(void *ptr) {
tsd_t *tsd = tsd_fetch_min();
check_entry_exit_locking(tsd_tsdn(tsd));
tcache_t *tcache;
if (likely(tsd_fast(tsd))) {
tsd_assert_fast(tsd);
/* Unconditionally get tcache ptr on fast path. */
tcache = tsd_tcachep_get(tsd);
ifree(tsd, ptr, tcache, false);
tcache_t *tcache = tcache_get_from_ind(tsd,
TCACHE_IND_AUTOMATIC, /* slow */ false,
/* is_alloc */ false);
ifree(tsd, ptr, tcache, /* slow */ false);
} else {
if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
tcache = tcache_get(tsd);
} else {
tcache = NULL;
}
tcache_t *tcache = tcache_get_from_ind(tsd,
TCACHE_IND_AUTOMATIC, /* slow */ true,
/* is_alloc */ false);
uintptr_t args_raw[3] = {(uintptr_t)ptr};
hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw);
ifree(tsd, ptr, tcache, true);
ifree(tsd, ptr, tcache, /* slow */ true);
}
check_entry_exit_locking(tsd_tsdn(tsd));
}
}
JEMALLOC_ALWAYS_INLINE bool
free_fastpath_nonfast_aligned(void *ptr, bool check_prof) {
/*
* free_fastpath do not handle two uncommon cases: 1) sampled profiled
* objects and 2) sampled junk & stash for use-after-free detection.
* Both have special alignments which are used to escape the fastpath.
*
* prof_sample is page-aligned, which covers the UAF check when both
* are enabled (the assertion below). Avoiding redundant checks since
* this is on the fastpath -- at most one runtime branch from this.
*/
if (config_debug && cache_bin_nonfast_aligned(ptr)) {
assert(prof_sample_aligned(ptr));
}
if (config_prof && check_prof) {
/* When prof is enabled, the prof_sample alignment is enough. */
if (prof_sample_aligned(ptr)) {
return true;
} else {
return false;
}
}
if (config_uaf_detection) {
if (cache_bin_nonfast_aligned(ptr)) {
return true;
} else {
return false;
}
}
return false;
}
/* Returns whether or not the free attempt was successful. */
JEMALLOC_ALWAYS_INLINE
bool free_fastpath(void *ptr, size_t size, bool size_hint) {
tsd_t *tsd = tsd_get(false);
if (unlikely(!tsd || !tsd_fast(tsd))) {
/* The branch gets optimized away unless tsd_get_allocates(). */
if (unlikely(tsd == NULL)) {
return false;
}
tcache_t *tcache = tsd_tcachep_get(tsd);
alloc_ctx_t alloc_ctx;
/*
* If !config_cache_oblivious, we can check PAGE alignment to
* detect sampled objects. Otherwise addresses are
* randomized, and we have to look it up in the rtree anyway.
* See also isfree().
* The tsd_fast() / initialized checks are folded into the branch
* testing (deallocated_after >= threshold) later in this function.
* The threshold will be set to 0 when !tsd_fast.
*/
if (!size_hint || config_cache_oblivious) {
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), &extents_rtree,
rtree_ctx, (uintptr_t)ptr,
&alloc_ctx.szind, &alloc_ctx.slab);
assert(tsd_fast(tsd) ||
*tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) == 0);
emap_alloc_ctx_t alloc_ctx;
if (!size_hint) {
bool err = emap_alloc_ctx_try_lookup_fast(tsd,
&arena_emap_global, ptr, &alloc_ctx);
/* Note: profiled objects will have alloc_ctx.slab set */
if (!res || !alloc_ctx.slab) {
if (unlikely(err || !alloc_ctx.slab ||
free_fastpath_nonfast_aligned(ptr,
/* check_prof */ false))) {
return false;
}
assert(alloc_ctx.szind != SC_NSIZES);
} else {
/*
* Check for both sizes that are too large, and for sampled objects.
* Sampled objects are always page-aligned. The sampled object check
* will also check for null ptr.
* Check for both sizes that are too large, and for sampled /
* special aligned objects. The alignment check will also check
* for null ptr.
*/
if (size > SC_LOOKUP_MAXCLASS || (((uintptr_t)ptr & PAGE_MASK) == 0)) {
if (unlikely(size > SC_LOOKUP_MAXCLASS ||
free_fastpath_nonfast_aligned(ptr,
/* check_prof */ true))) {
return false;
}
alloc_ctx.szind = sz_size2index_lookup(size);
/* Max lookup class must be small. */
assert(alloc_ctx.szind < SC_NBINS);
/* This is a dead store, except when opt size checking is on. */
alloc_ctx.slab = true;
}
/*
* Currently the fastpath only handles small sizes. The branch on
* SC_LOOKUP_MAXCLASS makes sure of it. This lets us avoid checking
* tcache szind upper limit (i.e. tcache_maxclass) as well.
*/
assert(alloc_ctx.slab);
uint64_t deallocated, threshold;
te_free_fastpath_ctx(tsd, &deallocated, &threshold);
if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
size_t usize = sz_index2size(alloc_ctx.szind);
uint64_t deallocated_after = deallocated + usize;
/*
* Check for events and tsd non-nominal (fast_threshold will be set to
* 0) in a single branch. Note that this handles the uninitialized case
* as well (TSD init will be triggered on the non-fastpath). Therefore
* anything depends on a functional TSD (e.g. the alloc_ctx sanity check
* below) needs to be after this branch.
*/
if (unlikely(deallocated_after >= threshold)) {
return false;
}
assert(tsd_fast(tsd));
bool fail = maybe_check_alloc_ctx(tsd, ptr, &alloc_ctx);
if (fail) {
/* See the comment in isfree. */
return true;
}
tcache_t *tcache = tcache_get_from_ind(tsd, TCACHE_IND_AUTOMATIC,
/* slow */ false, /* is_alloc */ false);
cache_bin_t *bin = &tcache->bins[alloc_ctx.szind];
/*
* If junking were enabled, this is where we would do it. It's not
* though, since we ensured above that we're on the fast path. Assert
* that to double-check.
*/
assert(!opt_junk_free);
cache_bin_t *bin = tcache_small_bin_get(tcache, alloc_ctx.szind);
cache_bin_info_t *bin_info = &tcache_bin_info[alloc_ctx.szind];
if (!cache_bin_dalloc_easy(bin, bin_info, ptr)) {
if (!cache_bin_dalloc_easy(bin, ptr)) {
return false;
}
if (config_stats) {
size_t usize = sz_index2size(alloc_ctx.szind);
*tsd_thread_deallocatedp_get(tsd) += usize;
}
*tsd_thread_deallocatedp_get(tsd) = deallocated_after;
return true;
}
......@@ -2965,6 +3260,8 @@ je_valloc(size_t size) {
* passed an extra argument for the caller return address, which will be
* ignored.
*/
#include <features.h> // defines __GLIBC__ if we are compiling against glibc
JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
......@@ -2973,7 +3270,7 @@ JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
je_memalign;
# endif
# ifdef CPU_COUNT
# ifdef __GLIBC__
/*
* To enable static linking with glibc, the libc specific malloc interface must
* be implemented also, so none of glibc's malloc.o functions are added to the
......@@ -3016,11 +3313,31 @@ int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign);
* Begin non-standard functions.
*/
#ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API
JEMALLOC_ALWAYS_INLINE unsigned
mallocx_tcache_get(int flags) {
if (likely((flags & MALLOCX_TCACHE_MASK) == 0)) {
return TCACHE_IND_AUTOMATIC;
} else if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
return TCACHE_IND_NONE;
} else {
return MALLOCX_TCACHE_GET(flags);
}
}
#define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y
#define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \
JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y)
JEMALLOC_ALWAYS_INLINE unsigned
mallocx_arena_get(int flags) {
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
return MALLOCX_ARENA_GET(flags);
} else {
return ARENA_IND_AUTOMATIC;
}
}
#ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API
#define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y
#define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \
JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y)
typedef struct {
void *ptr;
......@@ -3060,25 +3377,10 @@ JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT)
dopts.num_items = 1;
dopts.item_size = size;
if (unlikely(flags != 0)) {
if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) {
dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
}
dopts.alignment = MALLOCX_ALIGN_GET(flags);
dopts.zero = MALLOCX_ZERO_GET(flags);
if ((flags & MALLOCX_TCACHE_MASK) != 0) {
if ((flags & MALLOCX_TCACHE_MASK)
== MALLOCX_TCACHE_NONE) {
dopts.tcache_ind = TCACHE_IND_NONE;
} else {
dopts.tcache_ind = MALLOCX_TCACHE_GET(flags);
}
} else {
dopts.tcache_ind = TCACHE_IND_AUTOMATIC;
}
if ((flags & MALLOCX_ARENA_MASK) != 0)
dopts.arena_ind = MALLOCX_ARENA_GET(flags);
dopts.tcache_ind = mallocx_tcache_get(flags);
dopts.arena_ind = mallocx_arena_get(flags);
}
imalloc(&sopts, &dopts);
......@@ -3113,25 +3415,10 @@ je_mallocx(size_t size, int flags) {
dopts.num_items = 1;
dopts.item_size = size;
if (unlikely(flags != 0)) {
if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) {
dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
}
dopts.alignment = MALLOCX_ALIGN_GET(flags);
dopts.zero = MALLOCX_ZERO_GET(flags);
if ((flags & MALLOCX_TCACHE_MASK) != 0) {
if ((flags & MALLOCX_TCACHE_MASK)
== MALLOCX_TCACHE_NONE) {
dopts.tcache_ind = TCACHE_IND_NONE;
} else {
dopts.tcache_ind = MALLOCX_TCACHE_GET(flags);
}
} else {
dopts.tcache_ind = TCACHE_IND_AUTOMATIC;
}
if ((flags & MALLOCX_ARENA_MASK) != 0)
dopts.arena_ind = MALLOCX_ARENA_GET(flags);
dopts.tcache_ind = mallocx_tcache_get(flags);
dopts.arena_ind = mallocx_arena_get(flags);
}
imalloc(&sopts, &dopts);
......@@ -3154,6 +3441,8 @@ irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize,
if (tctx == NULL) {
return NULL;
}
alignment = prof_sample_align(alignment);
if (usize <= SC_SMALL_MAXCLASS) {
p = iralloct(tsdn, old_ptr, old_usize,
SC_LARGE_MINCLASS, alignment, zero, tcache,
......@@ -3166,66 +3455,48 @@ irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize,
p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero,
tcache, arena, hook_args);
}
assert(prof_sample_aligned(p));
return p;
}
JEMALLOC_ALWAYS_INLINE void *
irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
arena_t *arena, alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) {
size_t alignment, size_t usize, bool zero, tcache_t *tcache,
arena_t *arena, emap_alloc_ctx_t *alloc_ctx,
hook_ralloc_args_t *hook_args) {
prof_info_t old_prof_info;
prof_info_get_and_reset_recent(tsd, old_ptr, alloc_ctx, &old_prof_info);
bool prof_active = prof_active_get_unlocked();
bool sample_event = te_prof_sample_event_lookahead(tsd, usize);
prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active, sample_event);
void *p;
bool prof_active;
prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked();
old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx);
tctx = prof_alloc_prep(tsd, *usize, prof_active, false);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize,
*usize, alignment, zero, tcache, arena, tctx, hook_args);
usize, alignment, zero, tcache, arena, tctx, hook_args);
} else {
p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment,
zero, tcache, arena, hook_args);
}
if (unlikely(p == NULL)) {
prof_alloc_rollback(tsd, tctx, false);
prof_alloc_rollback(tsd, tctx);
return NULL;
}
if (p == old_ptr && alignment != 0) {
/*
* The allocation did not move, so it is possible that the size
* class is smaller than would guarantee the requested
* alignment, and that the alignment constraint was
* serendipitously satisfied. Additionally, old_usize may not
* be the same as the current usize because of in-place large
* reallocation. Therefore, query the actual value of usize.
*/
*usize = isalloc(tsd_tsdn(tsd), p);
}
prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr,
old_usize, old_tctx);
assert(usize == isalloc(tsd_tsdn(tsd), p));
prof_realloc(tsd, p, size, usize, tctx, prof_active, old_ptr,
old_usize, &old_prof_info, sample_event);
return p;
}
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ALLOC_SIZE(2)
je_rallocx(void *ptr, size_t size, int flags) {
static void *
do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
void *p;
tsd_t *tsd;
size_t usize;
size_t old_usize;
size_t alignment = MALLOCX_ALIGN_GET(flags);
bool zero = flags & MALLOCX_ZERO;
arena_t *arena;
tcache_t *tcache;
LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
size, flags);
assert(ptr != NULL);
assert(size != 0);
......@@ -3233,44 +3504,31 @@ je_rallocx(void *ptr, size_t size, int flags) {
tsd = tsd_fetch();
check_entry_exit_locking(tsd_tsdn(tsd));
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
unsigned arena_ind = MALLOCX_ARENA_GET(flags);
arena = arena_get(tsd_tsdn(tsd), arena_ind, true);
if (unlikely(arena == NULL)) {
goto label_oom;
}
} else {
arena = NULL;
}
bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true);
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
tcache = NULL;
} else {
tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
}
} else {
tcache = tcache_get(tsd);
unsigned arena_ind = mallocx_arena_get(flags);
if (arena_get_from_ind(tsd, arena_ind, &arena)) {
goto label_oom;
}
alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
unsigned tcache_ind = mallocx_tcache_get(flags);
tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind,
/* slow */ true, /* is_alloc */ true);
emap_alloc_ctx_t alloc_ctx;
emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
&alloc_ctx);
assert(alloc_ctx.szind != SC_NSIZES);
old_usize = sz_index2size(alloc_ctx.szind);
assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
if (aligned_usize_get(size, alignment, &usize, NULL, false)) {
goto label_oom;
}
hook_ralloc_args_t hook_args = {false, {(uintptr_t)ptr, size, flags,
0}};
hook_ralloc_args_t hook_args = {is_realloc, {(uintptr_t)ptr, size,
flags, 0}};
if (config_prof && opt_prof) {
usize = (alignment == 0) ?
sz_s2u(size) : sz_sa2u(size, alignment);
if (unlikely(usize == 0
|| usize > SC_LARGE_MAXCLASS)) {
goto label_oom;
}
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, usize,
zero, tcache, arena, &alloc_ctx, &hook_args);
if (unlikely(p == NULL)) {
goto label_oom;
......@@ -3281,20 +3539,22 @@ je_rallocx(void *ptr, size_t size, int flags) {
if (unlikely(p == NULL)) {
goto label_oom;
}
if (config_stats) {
usize = isalloc(tsd_tsdn(tsd), p);
}
assert(usize == isalloc(tsd_tsdn(tsd), p));
}
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
thread_alloc_event(tsd, usize);
thread_dalloc_event(tsd, old_usize);
if (config_stats) {
*tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize;
}
UTRACE(ptr, size, p);
check_entry_exit_locking(tsd_tsdn(tsd));
LOG("core.rallocx.exit", "result: %p", p);
if (config_fill && unlikely(opt_junk_alloc) && usize > old_usize
&& !zero) {
size_t excess_len = usize - old_usize;
void *excess_start = (void *)((uintptr_t)p + old_usize);
junk_alloc_callback(excess_start, excess_len);
}
return p;
label_oom:
if (config_xmalloc && unlikely(opt_xmalloc)) {
......@@ -3304,10 +3564,103 @@ label_oom:
UTRACE(ptr, size, 0);
check_entry_exit_locking(tsd_tsdn(tsd));
LOG("core.rallocx.exit", "result: %p", NULL);
return NULL;
}
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ALLOC_SIZE(2)
je_rallocx(void *ptr, size_t size, int flags) {
LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
size, flags);
void *ret = do_rallocx(ptr, size, flags, false);
LOG("core.rallocx.exit", "result: %p", ret);
return ret;
}
static void *
do_realloc_nonnull_zero(void *ptr) {
if (config_stats) {
atomic_fetch_add_zu(&zero_realloc_count, 1, ATOMIC_RELAXED);
}
if (opt_zero_realloc_action == zero_realloc_action_alloc) {
/*
* The user might have gotten an alloc setting while expecting a
* free setting. If that's the case, we at least try to
* reduce the harm, and turn off the tcache while allocating, so
* that we'll get a true first fit.
*/
return do_rallocx(ptr, 1, MALLOCX_TCACHE_NONE, true);
} else if (opt_zero_realloc_action == zero_realloc_action_free) {
UTRACE(ptr, 0, 0);
tsd_t *tsd = tsd_fetch();
check_entry_exit_locking(tsd_tsdn(tsd));
tcache_t *tcache = tcache_get_from_ind(tsd,
TCACHE_IND_AUTOMATIC, /* slow */ true,
/* is_alloc */ false);
uintptr_t args[3] = {(uintptr_t)ptr, 0};
hook_invoke_dalloc(hook_dalloc_realloc, ptr, args);
ifree(tsd, ptr, tcache, true);
check_entry_exit_locking(tsd_tsdn(tsd));
return NULL;
} else {
safety_check_fail("Called realloc(non-null-ptr, 0) with "
"zero_realloc:abort set\n");
/* In real code, this will never run; the safety check failure
* will call abort. In the unit test, we just want to bail out
* without corrupting internal state that the test needs to
* finish.
*/
return NULL;
}
}
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ALLOC_SIZE(2)
je_realloc(void *ptr, size_t size) {
LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);
if (likely(ptr != NULL && size != 0)) {
void *ret = do_rallocx(ptr, size, 0, true);
LOG("core.realloc.exit", "result: %p", ret);
return ret;
} else if (ptr != NULL && size == 0) {
void *ret = do_realloc_nonnull_zero(ptr);
LOG("core.realloc.exit", "result: %p", ret);
return ret;
} else {
/* realloc(NULL, size) is equivalent to malloc(size). */
void *ret;
static_opts_t sopts;
dynamic_opts_t dopts;
static_opts_init(&sopts);
dynamic_opts_init(&dopts);
sopts.null_out_result_on_error = true;
sopts.set_errno_on_error = true;
sopts.oom_string =
"<jemalloc>: Error in realloc(): out of memory\n";
dopts.result = &ret;
dopts.num_items = 1;
dopts.item_size = size;
imalloc(&sopts, &dopts);
if (sopts.slow) {
uintptr_t args[3] = {(uintptr_t)ptr, size};
hook_invoke_alloc(hook_alloc_realloc, ret,
(uintptr_t)ret, args);
}
LOG("core.realloc.exit", "result: %p", ret);
return ret;
}
}
JEMALLOC_ALWAYS_INLINE size_t
ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
size_t extra, size_t alignment, bool zero) {
......@@ -3324,51 +3677,46 @@ ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
static size_t
ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) {
size_t usize;
if (tctx == NULL) {
/* Sampled allocation needs to be page aligned. */
if (tctx == NULL || !prof_sample_aligned(ptr)) {
return old_usize;
}
usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
zero);
return usize;
return ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
zero);
}
JEMALLOC_ALWAYS_INLINE size_t
ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
size_t extra, size_t alignment, bool zero, alloc_ctx_t *alloc_ctx) {
size_t usize_max, usize;
bool prof_active;
prof_tctx_t *old_tctx, *tctx;
size_t extra, size_t alignment, bool zero, emap_alloc_ctx_t *alloc_ctx) {
/*
* old_prof_info is only used for asserting that the profiling info
* isn't changed by the ixalloc() call.
*/
prof_info_t old_prof_info;
prof_info_get(tsd, ptr, alloc_ctx, &old_prof_info);
prof_active = prof_active_get_unlocked();
old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr, alloc_ctx);
/*
* usize isn't knowable before ixalloc() returns when extra is non-zero.
* Therefore, compute its maximum possible value and use that in
* prof_alloc_prep() to decide whether to capture a backtrace.
* prof_realloc() will use the actual usize to decide whether to sample.
*/
if (alignment == 0) {
usize_max = sz_s2u(size+extra);
assert(usize_max > 0
&& usize_max <= SC_LARGE_MAXCLASS);
} else {
usize_max = sz_sa2u(size+extra, alignment);
if (unlikely(usize_max == 0
|| usize_max > SC_LARGE_MAXCLASS)) {
/*
* usize_max is out of range, and chances are that
* allocation will fail, but use the maximum possible
* value and carry on with prof_alloc_prep(), just in
* case allocation succeeds.
*/
usize_max = SC_LARGE_MAXCLASS;
}
size_t usize_max;
if (aligned_usize_get(size + extra, alignment, &usize_max, NULL,
false)) {
/*
* usize_max is out of range, and chances are that allocation
* will fail, but use the maximum possible value and carry on
* with prof_alloc_prep(), just in case allocation succeeds.
*/
usize_max = SC_LARGE_MAXCLASS;
}
tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
bool prof_active = prof_active_get_unlocked();
bool sample_event = te_prof_sample_event_lookahead(tsd, usize_max);
prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active, sample_event);
size_t usize;
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
size, extra, alignment, zero, tctx);
......@@ -3376,13 +3724,28 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
extra, alignment, zero);
}
/*
* At this point we can still safely get the original profiling
* information associated with the ptr, because (a) the edata_t object
* associated with the ptr still lives and (b) the profiling info
* fields are not touched. "(a)" is asserted in the outer je_xallocx()
* function, and "(b)" is indirectly verified below by checking that
* the alloc_tctx field is unchanged.
*/
prof_info_t prof_info;
if (usize == old_usize) {
prof_alloc_rollback(tsd, tctx, false);
return usize;
prof_info_get(tsd, ptr, alloc_ctx, &prof_info);
prof_alloc_rollback(tsd, tctx);
} else {
prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info);
assert(usize <= usize_max);
sample_event = te_prof_sample_event_lookahead(tsd, usize);
prof_realloc(tsd, ptr, size, usize, tctx, prof_active, ptr,
old_usize, &prof_info, sample_event);
}
prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
old_tctx);
assert(old_prof_info.alloc_tctx == prof_info.alloc_tctx);
return usize;
}
......@@ -3391,7 +3754,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
tsd_t *tsd;
size_t usize, old_usize;
size_t alignment = MALLOCX_ALIGN_GET(flags);
bool zero = flags & MALLOCX_ZERO;
bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true);
LOG("core.xallocx.entry", "ptr: %p, size: %zu, extra: %zu, "
"flags: %d", ptr, size, extra, flags);
......@@ -3403,10 +3766,17 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
tsd = tsd_fetch();
check_entry_exit_locking(tsd_tsdn(tsd));
alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
/*
* old_edata is only for verifying that xallocx() keeps the edata_t
* object associated with the ptr (though the content of the edata_t
* object can be changed).
*/
edata_t *old_edata = emap_edata_lookup(tsd_tsdn(tsd),
&arena_emap_global, ptr);
emap_alloc_ctx_t alloc_ctx;
emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
&alloc_ctx);
assert(alloc_ctx.szind != SC_NSIZES);
old_usize = sz_index2size(alloc_ctx.szind);
assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
......@@ -3434,13 +3804,25 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
extra, alignment, zero);
}
/*
* xallocx() should keep using the same edata_t object (though its
* content can be changed).
*/
assert(emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr)
== old_edata);
if (unlikely(usize == old_usize)) {
goto label_not_resized;
}
thread_alloc_event(tsd, usize);
thread_dalloc_event(tsd, old_usize);
if (config_stats) {
*tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize;
if (config_fill && unlikely(opt_junk_alloc) && usize > old_usize &&
!zero) {
size_t excess_len = usize - old_usize;
void *excess_start = (void *)((uintptr_t)ptr + old_usize);
junk_alloc_callback(excess_start, excess_len);
}
label_not_resized:
if (unlikely(!tsd_fast(tsd))) {
......@@ -3490,31 +3872,13 @@ je_dallocx(void *ptr, int flags) {
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
tsd_t *tsd = tsd_fetch();
tsd_t *tsd = tsd_fetch_min();
bool fast = tsd_fast(tsd);
check_entry_exit_locking(tsd_tsdn(tsd));
tcache_t *tcache;
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
/* Not allowed to be reentrant and specify a custom tcache. */
assert(tsd_reentrancy_level_get(tsd) == 0);
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
tcache = NULL;
} else {
tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
}
} else {
if (likely(fast)) {
tcache = tsd_tcachep_get(tsd);
assert(tcache == tcache_get(tsd));
} else {
if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
tcache = tcache_get(tsd);
} else {
tcache = NULL;
}
}
}
unsigned tcache_ind = mallocx_tcache_get(flags);
tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast,
/* is_alloc */ false);
UTRACE(ptr, 0, 0);
if (likely(fast)) {
......@@ -3533,13 +3897,9 @@ je_dallocx(void *ptr, int flags) {
JEMALLOC_ALWAYS_INLINE size_t
inallocx(tsdn_t *tsdn, size_t size, int flags) {
check_entry_exit_locking(tsdn);
size_t usize;
if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) {
usize = sz_s2u(size);
} else {
usize = sz_sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
}
/* In case of out of range, let the user see it rather than fail. */
aligned_usize_get(size, MALLOCX_ALIGN_GET(flags), &usize, NULL, false);
check_entry_exit_locking(tsdn);
return usize;
}
......@@ -3549,33 +3909,14 @@ sdallocx_default(void *ptr, size_t size, int flags) {
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
tsd_t *tsd = tsd_fetch();
tsd_t *tsd = tsd_fetch_min();
bool fast = tsd_fast(tsd);
size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
assert(usize == isalloc(tsd_tsdn(tsd), ptr));
check_entry_exit_locking(tsd_tsdn(tsd));
tcache_t *tcache;
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
/* Not allowed to be reentrant and specify a custom tcache. */
assert(tsd_reentrancy_level_get(tsd) == 0);
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
tcache = NULL;
} else {
tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
}
} else {
if (likely(fast)) {
tcache = tsd_tcachep_get(tsd);
assert(tcache == tcache_get(tsd));
} else {
if (likely(tsd_reentrancy_level_get(tsd) == 0)) {
tcache = tcache_get(tsd);
} else {
tcache = NULL;
}
}
}
unsigned tcache_ind = mallocx_tcache_get(flags);
tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast,
/* is_alloc */ false);
UTRACE(ptr, 0, 0);
if (likely(fast)) {
......@@ -3587,7 +3928,6 @@ sdallocx_default(void *ptr, size_t size, int flags) {
isfree(tsd, ptr, usize, tcache, true);
}
check_entry_exit_locking(tsd_tsdn(tsd));
}
JEMALLOC_EXPORT void JEMALLOC_NOTHROW
......@@ -3595,7 +3935,7 @@ je_sdallocx(void *ptr, size_t size, int flags) {
LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
size, flags);
if (flags !=0 || !free_fastpath(ptr, size, true)) {
if (flags != 0 || !free_fastpath(ptr, size, true)) {
sdallocx_default(ptr, size, flags);
}
......@@ -3704,6 +4044,7 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
return ret;
}
#define STATS_PRINT_BUFSIZE 65536
JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
const char *opts) {
......@@ -3713,23 +4054,30 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
tsdn = tsdn_fetch();
check_entry_exit_locking(tsdn);
stats_print(write_cb, cbopaque, opts);
if (config_debug) {
stats_print(write_cb, cbopaque, opts);
} else {
buf_writer_t buf_writer;
buf_writer_init(tsdn, &buf_writer, write_cb, cbopaque, NULL,
STATS_PRINT_BUFSIZE);
stats_print(buf_writer_cb, &buf_writer, opts);
buf_writer_terminate(tsdn, &buf_writer);
}
check_entry_exit_locking(tsdn);
LOG("core.malloc_stats_print.exit", "");
}
#undef STATS_PRINT_BUFSIZE
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
size_t ret;
tsdn_t *tsdn;
LOG("core.malloc_usable_size.entry", "ptr: %p", ptr);
JEMALLOC_ALWAYS_INLINE size_t
je_malloc_usable_size_impl(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
assert(malloc_initialized() || IS_INITIALIZER);
tsdn = tsdn_fetch();
tsdn_t *tsdn = tsdn_fetch();
check_entry_exit_locking(tsdn);
size_t ret;
if (unlikely(ptr == NULL)) {
ret = 0;
} else {
......@@ -3740,12 +4088,211 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
ret = isalloc(tsdn, ptr);
}
}
check_entry_exit_locking(tsdn);
return ret;
}
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
LOG("core.malloc_usable_size.entry", "ptr: %p", ptr);
size_t ret = je_malloc_usable_size_impl(ptr);
LOG("core.malloc_usable_size.exit", "result: %zu", ret);
return ret;
}
#ifdef JEMALLOC_HAVE_MALLOC_SIZE
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_malloc_size(const void *ptr) {
LOG("core.malloc_size.entry", "ptr: %p", ptr);
size_t ret = je_malloc_usable_size_impl(ptr);
LOG("core.malloc_size.exit", "result: %zu", ret);
return ret;
}
#endif
static void
batch_alloc_prof_sample_assert(tsd_t *tsd, size_t batch, size_t usize) {
assert(config_prof && opt_prof);
bool prof_sample_event = te_prof_sample_event_lookahead(tsd,
batch * usize);
assert(!prof_sample_event);
size_t surplus;
prof_sample_event = te_prof_sample_event_lookahead_surplus(tsd,
(batch + 1) * usize, &surplus);
assert(prof_sample_event);
assert(surplus < usize);
}
size_t
batch_alloc(void **ptrs, size_t num, size_t size, int flags) {
LOG("core.batch_alloc.entry",
"ptrs: %p, num: %zu, size: %zu, flags: %d", ptrs, num, size, flags);
tsd_t *tsd = tsd_fetch();
check_entry_exit_locking(tsd_tsdn(tsd));
size_t filled = 0;
if (unlikely(tsd == NULL || tsd_reentrancy_level_get(tsd) > 0)) {
goto label_done;
}
size_t alignment = MALLOCX_ALIGN_GET(flags);
size_t usize;
if (aligned_usize_get(size, alignment, &usize, NULL, false)) {
goto label_done;
}
szind_t ind = sz_size2index(usize);
bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true);
/*
* The cache bin and arena will be lazily initialized; it's hard to
* know in advance whether each of them needs to be initialized.
*/
cache_bin_t *bin = NULL;
arena_t *arena = NULL;
size_t nregs = 0;
if (likely(ind < SC_NBINS)) {
nregs = bin_infos[ind].nregs;
assert(nregs > 0);
}
while (filled < num) {
size_t batch = num - filled;
size_t surplus = SIZE_MAX; /* Dead store. */
bool prof_sample_event = config_prof && opt_prof
&& prof_active_get_unlocked()
&& te_prof_sample_event_lookahead_surplus(tsd,
batch * usize, &surplus);
if (prof_sample_event) {
/*
* Adjust so that the batch does not trigger prof
* sampling.
*/
batch -= surplus / usize + 1;
batch_alloc_prof_sample_assert(tsd, batch, usize);
}
size_t progress = 0;
if (likely(ind < SC_NBINS) && batch >= nregs) {
if (arena == NULL) {
unsigned arena_ind = mallocx_arena_get(flags);
if (arena_get_from_ind(tsd, arena_ind,
&arena)) {
goto label_done;
}
if (arena == NULL) {
arena = arena_choose(tsd, NULL);
}
if (unlikely(arena == NULL)) {
goto label_done;
}
}
size_t arena_batch = batch - batch % nregs;
size_t n = arena_fill_small_fresh(tsd_tsdn(tsd), arena,
ind, ptrs + filled, arena_batch, zero);
progress += n;
filled += n;
}
if (likely(ind < nhbins) && progress < batch) {
if (bin == NULL) {
unsigned tcache_ind = mallocx_tcache_get(flags);
tcache_t *tcache = tcache_get_from_ind(tsd,
tcache_ind, /* slow */ true,
/* is_alloc */ true);
if (tcache != NULL) {
bin = &tcache->bins[ind];
}
}
/*
* If we don't have a tcache bin, we don't want to
* immediately give up, because there's the possibility
* that the user explicitly requested to bypass the
* tcache, or that the user explicitly turned off the
* tcache; in such cases, we go through the slow path,
* i.e. the mallocx() call at the end of the while loop.
*/
if (bin != NULL) {
size_t bin_batch = batch - progress;
/*
* n can be less than bin_batch, meaning that
* the cache bin does not have enough memory.
* In such cases, we rely on the slow path,
* i.e. the mallocx() call at the end of the
* while loop, to fill in the cache, and in the
* next iteration of the while loop, the tcache
* will contain a lot of memory, and we can
* harvest them here. Compared to the
* alternative approach where we directly go to
* the arena bins here, the overhead of our
* current approach should usually be minimal,
* since we never try to fetch more memory than
* what a slab contains via the tcache. An
* additional benefit is that the tcache will
* not be empty for the next allocation request.
*/
size_t n = cache_bin_alloc_batch(bin, bin_batch,
ptrs + filled);
if (config_stats) {
bin->tstats.nrequests += n;
}
if (zero) {
for (size_t i = 0; i < n; ++i) {
memset(ptrs[filled + i], 0,
usize);
}
}
if (config_prof && opt_prof
&& unlikely(ind >= SC_NBINS)) {
for (size_t i = 0; i < n; ++i) {
prof_tctx_reset_sampled(tsd,
ptrs[filled + i]);
}
}
progress += n;
filled += n;
}
}
/*
* For thread events other than prof sampling, trigger them as
* if there's a single allocation of size (n * usize). This is
* fine because:
* (a) these events do not alter the allocation itself, and
* (b) it's possible that some event would have been triggered
* multiple times, instead of only once, if the allocations
* were handled individually, but it would do no harm (or
* even be beneficial) to coalesce the triggerings.
*/
thread_alloc_event(tsd, progress * usize);
if (progress < batch || prof_sample_event) {
void *p = je_mallocx(size, flags);
if (p == NULL) { /* OOM */
break;
}
if (progress == batch) {
assert(prof_sampled(tsd, p));
}
ptrs[filled++] = p;
}
}
label_done:
check_entry_exit_locking(tsd_tsdn(tsd));
LOG("core.batch_alloc.exit", "result: %zu", filled);
return filled;
}
/*
* End non-standard functions.
*/
......@@ -3812,7 +4359,7 @@ _malloc_prefork(void)
background_thread_prefork1(tsd_tsdn(tsd));
}
/* Break arena prefork into stages to preserve lock order. */
for (i = 0; i < 8; i++) {
for (i = 0; i < 9; i++) {
for (j = 0; j < narenas; j++) {
if ((arena = arena_get(tsd_tsdn(tsd), j, false)) !=
NULL) {
......@@ -3841,12 +4388,17 @@ _malloc_prefork(void)
case 7:
arena_prefork7(tsd_tsdn(tsd), arena);
break;
case 8:
arena_prefork8(tsd_tsdn(tsd), arena);
break;
default: not_reached();
}
}
}
}
prof_prefork1(tsd_tsdn(tsd));
stats_prefork(tsd_tsdn(tsd));
tsd_prefork(tsd);
}
......@@ -3874,6 +4426,7 @@ _malloc_postfork(void)
witness_postfork_parent(tsd_witness_tsdp_get(tsd));
/* Release all mutexes, now that fork() has completed. */
stats_postfork_parent(tsd_tsdn(tsd));
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena;
......@@ -3903,6 +4456,7 @@ jemalloc_postfork_child(void) {
witness_postfork_child(tsd_witness_tsdp_get(tsd));
/* Release all mutexes, now that fork() has completed. */
stats_postfork_child(tsd_tsdn(tsd));
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena;
......
......@@ -39,9 +39,29 @@ void operator delete(void *ptr, std::size_t size) noexcept;
void operator delete[](void *ptr, std::size_t size) noexcept;
#endif
#if __cpp_aligned_new >= 201606
/* C++17's over-aligned operators. */
void *operator new(std::size_t size, std::align_val_t);
void *operator new(std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept;
void *operator new[](std::size_t size, std::align_val_t);
void *operator new[](std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept;
void operator delete(void* ptr, std::align_val_t) noexcept;
void operator delete(void* ptr, std::align_val_t, const std::nothrow_t &) noexcept;
void operator delete(void* ptr, std::size_t size, std::align_val_t al) noexcept;
void operator delete[](void* ptr, std::align_val_t) noexcept;
void operator delete[](void* ptr, std::align_val_t, const std::nothrow_t &) noexcept;
void operator delete[](void* ptr, std::size_t size, std::align_val_t al) noexcept;
#endif
JEMALLOC_NOINLINE
static void *
handleOOM(std::size_t size, bool nothrow) {
if (opt_experimental_infallible_new) {
safety_check_fail("<jemalloc>: Allocation failed and "
"opt.experimental_infallible_new is true. Aborting.\n");
return nullptr;
}
void *ptr = nullptr;
while (ptr == nullptr) {
......@@ -71,15 +91,22 @@ handleOOM(std::size_t size, bool nothrow) {
return ptr;
}
template <bool IsNoExcept>
JEMALLOC_NOINLINE
static void *
fallback_impl(std::size_t size) noexcept(IsNoExcept) {
void *ptr = malloc_default(size);
if (likely(ptr != nullptr)) {
return ptr;
}
return handleOOM(size, IsNoExcept);
}
template <bool IsNoExcept>
JEMALLOC_ALWAYS_INLINE
void *
newImpl(std::size_t size) noexcept(IsNoExcept) {
void *ptr = je_malloc(size);
if (likely(ptr != nullptr))
return ptr;
return handleOOM(size, IsNoExcept);
return imalloc_fastpath(size, &fallback_impl<IsNoExcept>);
}
void *
......@@ -102,6 +129,42 @@ operator new[](std::size_t size, const std::nothrow_t &) noexcept {
return newImpl<true>(size);
}
#if __cpp_aligned_new >= 201606
template <bool IsNoExcept>
JEMALLOC_ALWAYS_INLINE
void *
alignedNewImpl(std::size_t size, std::align_val_t alignment) noexcept(IsNoExcept) {
void *ptr = je_aligned_alloc(static_cast<std::size_t>(alignment), size);
if (likely(ptr != nullptr)) {
return ptr;
}
return handleOOM(size, IsNoExcept);
}
void *
operator new(std::size_t size, std::align_val_t alignment) {
return alignedNewImpl<false>(size, alignment);
}
void *
operator new[](std::size_t size, std::align_val_t alignment) {
return alignedNewImpl<false>(size, alignment);
}
void *
operator new(std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept {
return alignedNewImpl<true>(size, alignment);
}
void *
operator new[](std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept {
return alignedNewImpl<true>(size, alignment);
}
#endif // __cpp_aligned_new
void
operator delete(void *ptr) noexcept {
je_free(ptr);
......@@ -123,19 +186,69 @@ void operator delete[](void *ptr, const std::nothrow_t &) noexcept {
#if __cpp_sized_deallocation >= 201309
JEMALLOC_ALWAYS_INLINE
void
operator delete(void *ptr, std::size_t size) noexcept {
sizedDeleteImpl(void* ptr, std::size_t size) noexcept {
if (unlikely(ptr == nullptr)) {
return;
}
je_sdallocx_noflags(ptr, size);
}
void operator delete[](void *ptr, std::size_t size) noexcept {
void
operator delete(void *ptr, std::size_t size) noexcept {
sizedDeleteImpl(ptr, size);
}
void
operator delete[](void *ptr, std::size_t size) noexcept {
sizedDeleteImpl(ptr, size);
}
#endif // __cpp_sized_deallocation
#if __cpp_aligned_new >= 201606
JEMALLOC_ALWAYS_INLINE
void
alignedSizedDeleteImpl(void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
if (config_debug) {
assert(((size_t)alignment & ((size_t)alignment - 1)) == 0);
}
if (unlikely(ptr == nullptr)) {
return;
}
je_sdallocx_noflags(ptr, size);
je_sdallocx(ptr, size, MALLOCX_ALIGN(alignment));
}
#endif // __cpp_sized_deallocation
void
operator delete(void* ptr, std::align_val_t) noexcept {
je_free(ptr);
}
void
operator delete[](void* ptr, std::align_val_t) noexcept {
je_free(ptr);
}
void
operator delete(void* ptr, std::align_val_t, const std::nothrow_t&) noexcept {
je_free(ptr);
}
void
operator delete[](void* ptr, std::align_val_t, const std::nothrow_t&) noexcept {
je_free(ptr);
}
void
operator delete(void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
alignedSizedDeleteImpl(ptr, size, alignment);
}
void
operator delete[](void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
alignedSizedDeleteImpl(ptr, size, alignment);
}
#endif // __cpp_aligned_new
#define JEMALLOC_LARGE_C_
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/prof_recent.h"
#include "jemalloc/internal/util.h"
/******************************************************************************/
......@@ -21,8 +21,7 @@ void *
large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
bool zero) {
size_t ausize;
extent_t *extent;
bool is_zeroed;
edata_t *edata;
UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false);
assert(!tsdn_null(tsdn) || arena != NULL);
......@@ -32,163 +31,80 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
return NULL;
}
if (config_fill && unlikely(opt_zero)) {
zero = true;
}
/*
* Copy zero into is_zeroed and pass the copy when allocating the
* extent, so that it is possible to make correct junk/zero fill
* decisions below, even if is_zeroed ends up true when zero is false.
*/
is_zeroed = zero;
if (likely(!tsdn_null(tsdn))) {
arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize);
}
if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn,
arena, usize, alignment, &is_zeroed)) == NULL) {
if (unlikely(arena == NULL) || (edata = arena_extent_alloc_large(tsdn,
arena, usize, alignment, zero)) == NULL) {
return NULL;
}
/* See comments in arena_bin_slabs_full_insert(). */
if (!arena_is_auto(arena)) {
/* Insert extent into large. */
/* Insert edata into large. */
malloc_mutex_lock(tsdn, &arena->large_mtx);
extent_list_append(&arena->large, extent);
edata_list_active_append(&arena->large, edata);
malloc_mutex_unlock(tsdn, &arena->large_mtx);
}
if (config_prof && arena_prof_accum(tsdn, arena, usize)) {
prof_idump(tsdn);
}
if (zero) {
assert(is_zeroed);
} else if (config_fill && unlikely(opt_junk_alloc)) {
memset(extent_addr_get(extent), JEMALLOC_ALLOC_JUNK,
extent_usize_get(extent));
}
arena_decay_tick(tsdn, arena);
return extent_addr_get(extent);
return edata_addr_get(edata);
}
static void
large_dalloc_junk_impl(void *ptr, size_t size) {
memset(ptr, JEMALLOC_FREE_JUNK, size);
}
large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk = large_dalloc_junk_impl;
static void
large_dalloc_maybe_junk_impl(void *ptr, size_t size) {
if (config_fill && have_dss && unlikely(opt_junk_free)) {
/*
* Only bother junk filling if the extent isn't about to be
* unmapped.
*/
if (opt_retain || (have_dss && extent_in_dss(ptr))) {
large_dalloc_junk(ptr, size);
}
}
}
large_dalloc_maybe_junk_t *JET_MUTABLE large_dalloc_maybe_junk =
large_dalloc_maybe_junk_impl;
static bool
large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {
arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent);
extent_hooks_t *extent_hooks = extent_hooks_get(arena);
size_t diff = extent_size_get(extent) - (usize + sz_large_pad);
large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) {
arena_t *arena = arena_get_from_edata(edata);
ehooks_t *ehooks = arena_get_ehooks(arena);
size_t old_size = edata_size_get(edata);
size_t old_usize = edata_usize_get(edata);
assert(oldusize > usize);
assert(old_usize > usize);
if (extent_hooks->split == NULL) {
if (ehooks_split_will_fail(ehooks)) {
return true;
}
/* Split excess pages. */
if (diff != 0) {
extent_t *trail = extent_split_wrapper(tsdn, arena,
&extent_hooks, extent, usize + sz_large_pad,
sz_size2index(usize), false, diff, SC_NSIZES, false);
if (trail == NULL) {
return true;
}
if (config_fill && unlikely(opt_junk_free)) {
large_dalloc_maybe_junk(extent_addr_get(trail),
extent_size_get(trail));
}
arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, trail);
bool deferred_work_generated = false;
bool err = pa_shrink(tsdn, &arena->pa_shard, edata, old_size,
usize + sz_large_pad, sz_size2index(usize),
&deferred_work_generated);
if (err) {
return true;
}
arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize);
if (deferred_work_generated) {
arena_handle_deferred_work(tsdn, arena);
}
arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize);
return false;
}
static bool
large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
bool zero) {
arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent);
extent_hooks_t *extent_hooks = extent_hooks_get(arena);
size_t trailsize = usize - oldusize;
arena_t *arena = arena_get_from_edata(edata);
if (extent_hooks->merge == NULL) {
return true;
}
size_t old_size = edata_size_get(edata);
size_t old_usize = edata_usize_get(edata);
size_t new_size = usize + sz_large_pad;
if (config_fill && unlikely(opt_zero)) {
zero = true;
}
/*
* Copy zero into is_zeroed_trail and pass the copy when allocating the
* extent, so that it is possible to make correct junk/zero fill
* decisions below, even if is_zeroed_trail ends up true when zero is
* false.
*/
bool is_zeroed_trail = zero;
bool commit = true;
extent_t *trail;
bool new_mapping;
if ((trail = extents_alloc(tsdn, arena, &extent_hooks,
&arena->extents_dirty, extent_past_get(extent), trailsize, 0,
CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL
|| (trail = extents_alloc(tsdn, arena, &extent_hooks,
&arena->extents_muzzy, extent_past_get(extent), trailsize, 0,
CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) {
if (config_stats) {
new_mapping = false;
}
} else {
if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks,
extent_past_get(extent), trailsize, 0, CACHELINE, false,
SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) {
return true;
}
if (config_stats) {
new_mapping = true;
}
}
szind_t szind = sz_size2index(usize);
if (extent_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) {
extent_dalloc_wrapper(tsdn, arena, &extent_hooks, trail);
return true;
bool deferred_work_generated = false;
bool err = pa_expand(tsdn, &arena->pa_shard, edata, old_size, new_size,
szind, zero, &deferred_work_generated);
if (deferred_work_generated) {
arena_handle_deferred_work(tsdn, arena);
}
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
szind_t szind = sz_size2index(usize);
extent_szind_set(extent, szind);
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)extent_addr_get(extent), szind, false);
if (config_stats && new_mapping) {
arena_stats_mapped_add(tsdn, &arena->stats, trailsize);
if (err) {
return true;
}
if (zero) {
if (config_cache_oblivious) {
if (opt_cache_oblivious) {
assert(sz_large_pad == PAGE);
/*
* Zero the trailing bytes of the original allocation's
* last page, since they are in an indeterminate state.
......@@ -197,28 +113,23 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
* of CACHELINE in [0 .. PAGE).
*/
void *zbase = (void *)
((uintptr_t)extent_addr_get(extent) + oldusize);
((uintptr_t)edata_addr_get(edata) + old_usize);
void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase +
PAGE));
size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase;
assert(nzero > 0);
memset(zbase, 0, nzero);
}
assert(is_zeroed_trail);
} else if (config_fill && unlikely(opt_junk_alloc)) {
memset((void *)((uintptr_t)extent_addr_get(extent) + oldusize),
JEMALLOC_ALLOC_JUNK, usize - oldusize);
}
arena_extent_ralloc_large_expand(tsdn, arena, extent, oldusize);
arena_extent_ralloc_large_expand(tsdn, arena, edata, old_usize);
return false;
}
bool
large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
size_t usize_max, bool zero) {
size_t oldusize = extent_usize_get(extent);
size_t oldusize = edata_usize_get(edata);
/* The following should have been caught by callers. */
assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS);
......@@ -228,16 +139,15 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
if (usize_max > oldusize) {
/* Attempt to expand the allocation in-place. */
if (!large_ralloc_no_move_expand(tsdn, extent, usize_max,
if (!large_ralloc_no_move_expand(tsdn, edata, usize_max,
zero)) {
arena_decay_tick(tsdn, extent_arena_get(extent));
arena_decay_tick(tsdn, arena_get_from_edata(edata));
return false;
}
/* Try again, this time with usize_min. */
if (usize_min < usize_max && usize_min > oldusize &&
large_ralloc_no_move_expand(tsdn, extent, usize_min,
zero)) {
arena_decay_tick(tsdn, extent_arena_get(extent));
large_ralloc_no_move_expand(tsdn, edata, usize_min, zero)) {
arena_decay_tick(tsdn, arena_get_from_edata(edata));
return false;
}
}
......@@ -247,14 +157,14 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
* the new size.
*/
if (oldusize >= usize_min && oldusize <= usize_max) {
arena_decay_tick(tsdn, extent_arena_get(extent));
arena_decay_tick(tsdn, arena_get_from_edata(edata));
return false;
}
/* Attempt to shrink the allocation in-place. */
if (oldusize > usize_max) {
if (!large_ralloc_no_move_shrink(tsdn, extent, usize_max)) {
arena_decay_tick(tsdn, extent_arena_get(extent));
if (!large_ralloc_no_move_shrink(tsdn, edata, usize_max)) {
arena_decay_tick(tsdn, arena_get_from_edata(edata));
return false;
}
}
......@@ -274,9 +184,9 @@ void *
large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
size_t alignment, bool zero, tcache_t *tcache,
hook_ralloc_args_t *hook_args) {
extent_t *extent = iealloc(tsdn, ptr);
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
size_t oldusize = extent_usize_get(extent);
size_t oldusize = edata_usize_get(edata);
/* The following should have been caught by callers. */
assert(usize > 0 && usize <= SC_LARGE_MAXCLASS);
/* Both allocation sizes must be large to avoid a move. */
......@@ -284,11 +194,11 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
&& usize >= SC_LARGE_MINCLASS);
/* Try to avoid moving the allocation. */
if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) {
if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) {
hook_invoke_expand(hook_args->is_realloc
? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize,
usize, (uintptr_t)ptr, hook_args->args);
return extent_addr_get(extent);
return edata_addr_get(edata);
}
/*
......@@ -309,87 +219,104 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args);
size_t copysize = (usize < oldusize) ? usize : oldusize;
memcpy(ret, extent_addr_get(extent), copysize);
isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true);
memcpy(ret, edata_addr_get(edata), copysize);
isdalloct(tsdn, edata_addr_get(edata), oldusize, tcache, NULL, true);
return ret;
}
/*
* junked_locked indicates whether the extent's data have been junk-filled, and
* whether the arena's large_mtx is currently held.
* locked indicates whether the arena's large_mtx is currently held.
*/
static void
large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
bool junked_locked) {
if (!junked_locked) {
large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
bool locked) {
if (!locked) {
/* See comments in arena_bin_slabs_full_insert(). */
if (!arena_is_auto(arena)) {
malloc_mutex_lock(tsdn, &arena->large_mtx);
extent_list_remove(&arena->large, extent);
edata_list_active_remove(&arena->large, edata);
malloc_mutex_unlock(tsdn, &arena->large_mtx);
}
large_dalloc_maybe_junk(extent_addr_get(extent),
extent_usize_get(extent));
} else {
/* Only hold the large_mtx if necessary. */
if (!arena_is_auto(arena)) {
malloc_mutex_assert_owner(tsdn, &arena->large_mtx);
extent_list_remove(&arena->large, extent);
edata_list_active_remove(&arena->large, edata);
}
}
arena_extent_dalloc_large_prep(tsdn, arena, extent);
arena_extent_dalloc_large_prep(tsdn, arena, edata);
}
static void
large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, extent);
large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
bool deferred_work_generated = false;
pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated);
if (deferred_work_generated) {
arena_handle_deferred_work(tsdn, arena);
}
}
void
large_dalloc_prep_junked_locked(tsdn_t *tsdn, extent_t *extent) {
large_dalloc_prep_impl(tsdn, extent_arena_get(extent), extent, true);
large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata) {
large_dalloc_prep_impl(tsdn, arena_get_from_edata(edata), edata, true);
}
void
large_dalloc_finish(tsdn_t *tsdn, extent_t *extent) {
large_dalloc_finish_impl(tsdn, extent_arena_get(extent), extent);
large_dalloc_finish(tsdn_t *tsdn, edata_t *edata) {
large_dalloc_finish_impl(tsdn, arena_get_from_edata(edata), edata);
}
void
large_dalloc(tsdn_t *tsdn, extent_t *extent) {
arena_t *arena = extent_arena_get(extent);
large_dalloc_prep_impl(tsdn, arena, extent, false);
large_dalloc_finish_impl(tsdn, arena, extent);
large_dalloc(tsdn_t *tsdn, edata_t *edata) {
arena_t *arena = arena_get_from_edata(edata);
large_dalloc_prep_impl(tsdn, arena, edata, false);
large_dalloc_finish_impl(tsdn, arena, edata);
arena_decay_tick(tsdn, arena);
}
size_t
large_salloc(tsdn_t *tsdn, const extent_t *extent) {
return extent_usize_get(extent);
}
prof_tctx_t *
large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent) {
return extent_prof_tctx_get(extent);
large_salloc(tsdn_t *tsdn, const edata_t *edata) {
return edata_usize_get(edata);
}
void
large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx) {
extent_prof_tctx_set(extent, tctx);
large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info,
bool reset_recent) {
assert(prof_info != NULL);
prof_tctx_t *alloc_tctx = edata_prof_tctx_get(edata);
prof_info->alloc_tctx = alloc_tctx;
if ((uintptr_t)alloc_tctx > (uintptr_t)1U) {
nstime_copy(&prof_info->alloc_time,
edata_prof_alloc_time_get(edata));
prof_info->alloc_size = edata_prof_alloc_size_get(edata);
if (reset_recent) {
/*
* Reset the pointer on the recent allocation record,
* so that this allocation is recorded as released.
*/
prof_recent_alloc_reset(tsd, edata);
}
}
}
void
large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent) {
large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U);
static void
large_prof_tctx_set(edata_t *edata, prof_tctx_t *tctx) {
edata_prof_tctx_set(edata, tctx);
}
nstime_t
large_prof_alloc_time_get(const extent_t *extent) {
return extent_prof_alloc_time_get(extent);
void
large_prof_tctx_reset(edata_t *edata) {
large_prof_tctx_set(edata, (prof_tctx_t *)(uintptr_t)1U);
}
void
large_prof_alloc_time_set(extent_t *extent, nstime_t t) {
extent_prof_alloc_time_set(extent, t);
large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size) {
nstime_t t;
nstime_prof_init_update(&t);
edata_prof_alloc_time_set(edata, &t);
edata_prof_alloc_size_set(edata, size);
edata_prof_recent_alloc_init(edata);
large_prof_tctx_set(edata, tctx);
}
#define JEMALLOC_MALLOC_IO_C_
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
......@@ -53,7 +52,6 @@
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static void wrtmessage(void *cbopaque, const char *s);
#define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1)
static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s,
size_t *slen_p);
......@@ -68,7 +66,7 @@ static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
/******************************************************************************/
/* malloc_message() setup. */
static void
void
wrtmessage(void *cbopaque, const char *s) {
malloc_write_fd(STDERR_FILENO, s, strlen(s));
}
......@@ -135,10 +133,10 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) {
break;
case '-':
neg = true;
/* Fall through. */
JEMALLOC_FALLTHROUGH;
case '+':
p++;
/* Fall through. */
JEMALLOC_FALLTHROUGH;
default:
goto label_prefix;
}
......@@ -289,7 +287,7 @@ d2s(intmax_t x, char sign, char *s, size_t *slen_p) {
if (!neg) {
break;
}
/* Fall through. */
JEMALLOC_FALLTHROUGH;
case ' ':
case '+':
s--;
......@@ -323,6 +321,7 @@ x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) {
return s;
}
JEMALLOC_COLD
size_t
malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
size_t i;
......@@ -348,7 +347,11 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
if (!left_justify && pad_len != 0) { \
size_t j; \
for (j = 0; j < pad_len; j++) { \
APPEND_C(' '); \
if (pad_zero) { \
APPEND_C('0'); \
} else { \
APPEND_C(' '); \
} \
} \
} \
/* Value. */ \
......@@ -420,6 +423,8 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
unsigned char len = '?';
char *s;
size_t slen;
bool first_width_digit = true;
bool pad_zero = false;
f++;
/* Flags. */
......@@ -456,7 +461,12 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
width = -width;
}
break;
case '0': case '1': case '2': case '3': case '4':
case '0':
if (first_width_digit) {
pad_zero = true;
}
JEMALLOC_FALLTHROUGH;
case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': {
uintmax_t uwidth;
set_errno(0);
......@@ -464,6 +474,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
assert(uwidth != UINTMAX_MAX || get_errno() !=
ERANGE);
width = (int)uwidth;
first_width_digit = false;
break;
} default:
break;
......@@ -521,6 +532,18 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
char buf[D2S_BUFSIZE];
/*
* Outputting negative, zero-padded numbers
* would require a nontrivial rework of the
* interaction between the width and padding
* (since 0 padding goes between the '-' and the
* number, while ' ' padding goes either before
* the - or after the number. Since we
* currently don't ever need 0-padded negative
* numbers, just don't bother supporting it.
*/
assert(!pad_zero);
GET_ARG_NUMERIC(val, len);
s = d2s(val, (plus_plus ? '+' : (plus_space ?
' ' : '-')), buf, &slen);
......@@ -620,8 +643,8 @@ malloc_snprintf(char *str, size_t size, const char *format, ...) {
}
void
malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
const char *format, va_list ap) {
malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format,
va_list ap) {
char buf[MALLOC_PRINTF_BUFSIZE];
if (write_cb == NULL) {
......@@ -644,8 +667,7 @@ malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
*/
JEMALLOC_FORMAT_PRINTF(3, 4)
void
malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
const char *format, ...) {
malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format, ...) {
va_list ap;
va_start(ap, format);
......
#define JEMALLOC_MUTEX_C_
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
......@@ -10,6 +9,12 @@
#define _CRT_SPINCOUNT 4000
#endif
/*
* Based on benchmark results, a fixed spin with this amount of retries works
* well for our critical sections.
*/
int64_t opt_mutex_max_spin = 600;
/******************************************************************************/
/* Data. */
......@@ -46,13 +51,13 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
void
malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
mutex_prof_data_t *data = &mutex->prof_data;
nstime_t before = NSTIME_ZERO_INITIALIZER;
nstime_t before;
if (ncpus == 1) {
goto label_spin_done;
}
int cnt = 0, max_cnt = MALLOC_MUTEX_MAX_SPIN;
int cnt = 0;
do {
spin_cpu_spinwait();
if (!atomic_load_b(&mutex->locked, ATOMIC_RELAXED)
......@@ -60,7 +65,7 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
data->n_spin_acquired++;
return;
}
} while (cnt++ < max_cnt);
} while (cnt++ < opt_mutex_max_spin || opt_mutex_max_spin == -1);
if (!config_stats) {
/* Only spin is useful when stats is off. */
......@@ -68,7 +73,7 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
return;
}
label_spin_done:
nstime_update(&before);
nstime_init_update(&before);
/* Copy before to after to avoid clock skews. */
nstime_t after;
nstime_copy(&after, &before);
......@@ -104,8 +109,8 @@ label_spin_done:
static void
mutex_prof_data_init(mutex_prof_data_t *data) {
memset(data, 0, sizeof(mutex_prof_data_t));
nstime_init(&data->max_wait_time, 0);
nstime_init(&data->tot_wait_time, 0);
nstime_init_zero(&data->max_wait_time);
nstime_init_zero(&data->tot_wait_time);
data->prev_owner = NULL;
}
......
#define JEMALLOC_MUTEX_POOL_C_
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mutex_pool.h"
bool
mutex_pool_init(mutex_pool_t *pool, const char *name, witness_rank_t rank) {
for (int i = 0; i < MUTEX_POOL_SIZE; ++i) {
if (malloc_mutex_init(&pool->mutexes[i], name, rank,
malloc_mutex_address_ordered)) {
return true;
}
}
return false;
}
......@@ -8,96 +8,169 @@
#define BILLION UINT64_C(1000000000)
#define MILLION UINT64_C(1000000)
static void
nstime_set_initialized(nstime_t *time) {
#ifdef JEMALLOC_DEBUG
time->magic = NSTIME_MAGIC;
#endif
}
static void
nstime_assert_initialized(const nstime_t *time) {
#ifdef JEMALLOC_DEBUG
/*
* Some parts (e.g. stats) rely on memset to zero initialize. Treat
* these as valid initialization.
*/
assert(time->magic == NSTIME_MAGIC ||
(time->magic == 0 && time->ns == 0));
#endif
}
static void
nstime_pair_assert_initialized(const nstime_t *t1, const nstime_t *t2) {
nstime_assert_initialized(t1);
nstime_assert_initialized(t2);
}
static void
nstime_initialize_operand(nstime_t *time) {
/*
* Operations like nstime_add may have the initial operand being zero
* initialized (covered by the assert below). Full-initialize needed
* before changing it to non-zero.
*/
nstime_assert_initialized(time);
nstime_set_initialized(time);
}
void
nstime_init(nstime_t *time, uint64_t ns) {
nstime_set_initialized(time);
time->ns = ns;
}
void
nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec) {
nstime_set_initialized(time);
time->ns = sec * BILLION + nsec;
}
uint64_t
nstime_ns(const nstime_t *time) {
nstime_assert_initialized(time);
return time->ns;
}
uint64_t
nstime_msec(const nstime_t *time) {
nstime_assert_initialized(time);
return time->ns / MILLION;
}
uint64_t
nstime_sec(const nstime_t *time) {
nstime_assert_initialized(time);
return time->ns / BILLION;
}
uint64_t
nstime_nsec(const nstime_t *time) {
nstime_assert_initialized(time);
return time->ns % BILLION;
}
void
nstime_copy(nstime_t *time, const nstime_t *source) {
/* Source is required to be initialized. */
nstime_assert_initialized(source);
*time = *source;
nstime_assert_initialized(time);
}
int
nstime_compare(const nstime_t *a, const nstime_t *b) {
nstime_pair_assert_initialized(a, b);
return (a->ns > b->ns) - (a->ns < b->ns);
}
void
nstime_add(nstime_t *time, const nstime_t *addend) {
nstime_pair_assert_initialized(time, addend);
assert(UINT64_MAX - time->ns >= addend->ns);
nstime_initialize_operand(time);
time->ns += addend->ns;
}
void
nstime_iadd(nstime_t *time, uint64_t addend) {
nstime_assert_initialized(time);
assert(UINT64_MAX - time->ns >= addend);
nstime_initialize_operand(time);
time->ns += addend;
}
void
nstime_subtract(nstime_t *time, const nstime_t *subtrahend) {
nstime_pair_assert_initialized(time, subtrahend);
assert(nstime_compare(time, subtrahend) >= 0);
/* No initialize operand -- subtraction must be initialized. */
time->ns -= subtrahend->ns;
}
void
nstime_isubtract(nstime_t *time, uint64_t subtrahend) {
nstime_assert_initialized(time);
assert(time->ns >= subtrahend);
/* No initialize operand -- subtraction must be initialized. */
time->ns -= subtrahend;
}
void
nstime_imultiply(nstime_t *time, uint64_t multiplier) {
nstime_assert_initialized(time);
assert((((time->ns | multiplier) & (UINT64_MAX << (sizeof(uint64_t) <<
2))) == 0) || ((time->ns * multiplier) / multiplier == time->ns));
nstime_initialize_operand(time);
time->ns *= multiplier;
}
void
nstime_idivide(nstime_t *time, uint64_t divisor) {
nstime_assert_initialized(time);
assert(divisor != 0);
nstime_initialize_operand(time);
time->ns /= divisor;
}
uint64_t
nstime_divide(const nstime_t *time, const nstime_t *divisor) {
nstime_pair_assert_initialized(time, divisor);
assert(divisor->ns != 0);
/* No initialize operand -- *time itself remains unchanged. */
return time->ns / divisor->ns;
}
/* Returns time since *past, w/o updating *past. */
uint64_t
nstime_ns_since(const nstime_t *past) {
nstime_assert_initialized(past);
nstime_t now;
nstime_copy(&now, past);
nstime_update(&now);
assert(nstime_compare(&now, past) >= 0);
return now.ns - past->ns;
}
#ifdef _WIN32
# define NSTIME_MONOTONIC true
static void
......@@ -152,7 +225,42 @@ nstime_monotonic_impl(void) {
}
nstime_monotonic_t *JET_MUTABLE nstime_monotonic = nstime_monotonic_impl;
static bool
prof_time_res_t opt_prof_time_res =
prof_time_res_default;
const char *prof_time_res_mode_names[] = {
"default",
"high",
};
static void
nstime_get_realtime(nstime_t *time) {
#if defined(JEMALLOC_HAVE_CLOCK_REALTIME) && !defined(_WIN32)
struct timespec ts;
clock_gettime(CLOCK_REALTIME, &ts);
nstime_init2(time, ts.tv_sec, ts.tv_nsec);
#else
unreachable();
#endif
}
static void
nstime_prof_update_impl(nstime_t *time) {
nstime_t old_time;
nstime_copy(&old_time, time);
if (opt_prof_time_res == prof_time_res_high) {
nstime_get_realtime(time);
} else {
nstime_get(time);
}
}
nstime_prof_update_t *JET_MUTABLE nstime_prof_update = nstime_prof_update_impl;
static void
nstime_update_impl(nstime_t *time) {
nstime_t old_time;
......@@ -162,9 +270,20 @@ nstime_update_impl(nstime_t *time) {
/* Handle non-monotonic clocks. */
if (unlikely(nstime_compare(&old_time, time) > 0)) {
nstime_copy(time, &old_time);
return true;
}
return false;
}
nstime_update_t *JET_MUTABLE nstime_update = nstime_update_impl;
void
nstime_init_update(nstime_t *time) {
nstime_init_zero(time);
nstime_update(time);
}
void
nstime_prof_init_update(nstime_t *time) {
nstime_init_zero(time);
nstime_prof_update(time);
}
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/san.h"
#include "jemalloc/internal/hpa.h"
static void
pa_nactive_add(pa_shard_t *shard, size_t add_pages) {
atomic_fetch_add_zu(&shard->nactive, add_pages, ATOMIC_RELAXED);
}
static void
pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) {
assert(atomic_load_zu(&shard->nactive, ATOMIC_RELAXED) >= sub_pages);
atomic_fetch_sub_zu(&shard->nactive, sub_pages, ATOMIC_RELAXED);
}
bool
pa_central_init(pa_central_t *central, base_t *base, bool hpa,
hpa_hooks_t *hpa_hooks) {
bool err;
if (hpa) {
err = hpa_central_init(&central->hpa, base, hpa_hooks);
if (err) {
return true;
}
}
return false;
}
bool
pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats,
malloc_mutex_t *stats_mtx, nstime_t *cur_time,
size_t pac_oversize_threshold, ssize_t dirty_decay_ms,
ssize_t muzzy_decay_ms) {
/* This will change eventually, but for now it should hold. */
assert(base_ind_get(base) == ind);
if (edata_cache_init(&shard->edata_cache, base)) {
return true;
}
if (pac_init(tsdn, &shard->pac, base, emap, &shard->edata_cache,
cur_time, pac_oversize_threshold, dirty_decay_ms, muzzy_decay_ms,
&stats->pac_stats, stats_mtx)) {
return true;
}
shard->ind = ind;
shard->ever_used_hpa = false;
atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED);
atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED);
shard->stats_mtx = stats_mtx;
shard->stats = stats;
memset(shard->stats, 0, sizeof(*shard->stats));
shard->central = central;
shard->emap = emap;
shard->base = base;
return false;
}
bool
pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) {
if (hpa_shard_init(&shard->hpa_shard, &shard->central->hpa, shard->emap,
shard->base, &shard->edata_cache, shard->ind, hpa_opts)) {
return true;
}
if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai,
hpa_sec_opts)) {
return true;
}
shard->ever_used_hpa = true;
atomic_store_b(&shard->use_hpa, true, ATOMIC_RELAXED);
return false;
}
void
pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED);
if (shard->ever_used_hpa) {
sec_disable(tsdn, &shard->hpa_sec);
hpa_shard_disable(tsdn, &shard->hpa_shard);
}
}
void
pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED);
if (shard->ever_used_hpa) {
sec_flush(tsdn, &shard->hpa_sec);
}
}
static bool
pa_shard_uses_hpa(pa_shard_t *shard) {
return atomic_load_b(&shard->use_hpa, ATOMIC_RELAXED);
}
void
pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
pac_destroy(tsdn, &shard->pac);
if (shard->ever_used_hpa) {
sec_flush(tsdn, &shard->hpa_sec);
hpa_shard_disable(tsdn, &shard->hpa_shard);
}
}
static pai_t *
pa_get_pai(pa_shard_t *shard, edata_t *edata) {
return (edata_pai_get(edata) == EXTENT_PAI_PAC
? &shard->pac.pai : &shard->hpa_sec.pai);
}
edata_t *
pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
bool slab, szind_t szind, bool zero, bool guarded,
bool *deferred_work_generated) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
assert(!guarded || alignment <= PAGE);
edata_t *edata = NULL;
if (!guarded && pa_shard_uses_hpa(shard)) {
edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment,
zero, /* guarded */ false, slab, deferred_work_generated);
}
/*
* Fall back to the PAC if the HPA is off or couldn't serve the given
* allocation request.
*/
if (edata == NULL) {
edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero,
guarded, slab, deferred_work_generated);
}
if (edata != NULL) {
assert(edata_size_get(edata) == size);
pa_nactive_add(shard, size >> LG_PAGE);
emap_remap(tsdn, shard->emap, edata, szind, slab);
edata_szind_set(edata, szind);
edata_slab_set(edata, slab);
if (slab && (size > 2 * PAGE)) {
emap_register_interior(tsdn, shard->emap, edata, szind);
}
assert(edata_arena_ind_get(edata) == shard->ind);
}
return edata;
}
bool
pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
size_t new_size, szind_t szind, bool zero, bool *deferred_work_generated) {
assert(new_size > old_size);
assert(edata_size_get(edata) == old_size);
assert((new_size & PAGE_MASK) == 0);
if (edata_guarded_get(edata)) {
return true;
}
size_t expand_amount = new_size - old_size;
pai_t *pai = pa_get_pai(shard, edata);
bool error = pai_expand(tsdn, pai, edata, old_size, new_size, zero,
deferred_work_generated);
if (error) {
return true;
}
pa_nactive_add(shard, expand_amount >> LG_PAGE);
edata_szind_set(edata, szind);
emap_remap(tsdn, shard->emap, edata, szind, /* slab */ false);
return false;
}
bool
pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
size_t new_size, szind_t szind, bool *deferred_work_generated) {
assert(new_size < old_size);
assert(edata_size_get(edata) == old_size);
assert((new_size & PAGE_MASK) == 0);
if (edata_guarded_get(edata)) {
return true;
}
size_t shrink_amount = old_size - new_size;
pai_t *pai = pa_get_pai(shard, edata);
bool error = pai_shrink(tsdn, pai, edata, old_size, new_size,
deferred_work_generated);
if (error) {
return true;
}
pa_nactive_sub(shard, shrink_amount >> LG_PAGE);
edata_szind_set(edata, szind);
emap_remap(tsdn, shard->emap, edata, szind, /* slab */ false);
return false;
}
void
pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
bool *deferred_work_generated) {
emap_remap(tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false);
if (edata_slab_get(edata)) {
emap_deregister_interior(tsdn, shard->emap, edata);
/*
* The slab state of the extent isn't cleared. It may be used
* by the pai implementation, e.g. to make caching decisions.
*/
}
edata_addr_set(edata, edata_base_get(edata));
edata_szind_set(edata, SC_NSIZES);
pa_nactive_sub(shard, edata_size_get(edata) >> LG_PAGE);
pai_t *pai = pa_get_pai(shard, edata);
pai_dalloc(tsdn, pai, edata, deferred_work_generated);
}
bool
pa_shard_retain_grow_limit_get_set(tsdn_t *tsdn, pa_shard_t *shard,
size_t *old_limit, size_t *new_limit) {
return pac_retain_grow_limit_get_set(tsdn, &shard->pac, old_limit,
new_limit);
}
bool
pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state,
ssize_t decay_ms, pac_purge_eagerness_t eagerness) {
return pac_decay_ms_set(tsdn, &shard->pac, state, decay_ms, eagerness);
}
ssize_t
pa_decay_ms_get(pa_shard_t *shard, extent_state_t state) {
return pac_decay_ms_get(&shard->pac, state);
}
void
pa_shard_set_deferral_allowed(tsdn_t *tsdn, pa_shard_t *shard,
bool deferral_allowed) {
if (pa_shard_uses_hpa(shard)) {
hpa_shard_set_deferral_allowed(tsdn, &shard->hpa_shard,
deferral_allowed);
}
}
void
pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
if (pa_shard_uses_hpa(shard)) {
hpa_shard_do_deferred_work(tsdn, &shard->hpa_shard);
}
}
/*
* Get time until next deferred work ought to happen. If there are multiple
* things that have been deferred, this function calculates the time until
* the soonest of those things.
*/
uint64_t
pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
uint64_t time = pai_time_until_deferred_work(tsdn, &shard->pac.pai);
if (time == BACKGROUND_THREAD_DEFERRED_MIN) {
return time;
}
if (pa_shard_uses_hpa(shard)) {
uint64_t hpa =
pai_time_until_deferred_work(tsdn, &shard->hpa_shard.pai);
if (hpa < time) {
time = hpa;
}
}
return time;
}
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
/*
* This file is logically part of the PA module. While pa.c contains the core
* allocator functionality, this file contains boring integration functionality;
* things like the pre- and post- fork handlers, and stats merging for CTL
* refreshes.
*/
void
pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
malloc_mutex_prefork(tsdn, &shard->pac.decay_dirty.mtx);
malloc_mutex_prefork(tsdn, &shard->pac.decay_muzzy.mtx);
}
void
pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
if (shard->ever_used_hpa) {
sec_prefork2(tsdn, &shard->hpa_sec);
}
}
void
pa_shard_prefork3(tsdn_t *tsdn, pa_shard_t *shard) {
malloc_mutex_prefork(tsdn, &shard->pac.grow_mtx);
if (shard->ever_used_hpa) {
hpa_shard_prefork3(tsdn, &shard->hpa_shard);
}
}
void
pa_shard_prefork4(tsdn_t *tsdn, pa_shard_t *shard) {
ecache_prefork(tsdn, &shard->pac.ecache_dirty);
ecache_prefork(tsdn, &shard->pac.ecache_muzzy);
ecache_prefork(tsdn, &shard->pac.ecache_retained);
if (shard->ever_used_hpa) {
hpa_shard_prefork4(tsdn, &shard->hpa_shard);
}
}
void
pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard) {
edata_cache_prefork(tsdn, &shard->edata_cache);
}
void
pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
edata_cache_postfork_parent(tsdn, &shard->edata_cache);
ecache_postfork_parent(tsdn, &shard->pac.ecache_dirty);
ecache_postfork_parent(tsdn, &shard->pac.ecache_muzzy);
ecache_postfork_parent(tsdn, &shard->pac.ecache_retained);
malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx);
malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
if (shard->ever_used_hpa) {
sec_postfork_parent(tsdn, &shard->hpa_sec);
hpa_shard_postfork_parent(tsdn, &shard->hpa_shard);
}
}
void
pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
edata_cache_postfork_child(tsdn, &shard->edata_cache);
ecache_postfork_child(tsdn, &shard->pac.ecache_dirty);
ecache_postfork_child(tsdn, &shard->pac.ecache_muzzy);
ecache_postfork_child(tsdn, &shard->pac.ecache_retained);
malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx);
malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
if (shard->ever_used_hpa) {
sec_postfork_child(tsdn, &shard->hpa_sec);
hpa_shard_postfork_child(tsdn, &shard->hpa_shard);
}
}
void
pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, size_t *ndirty,
size_t *nmuzzy) {
*nactive += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
*ndirty += ecache_npages_get(&shard->pac.ecache_dirty);
*nmuzzy += ecache_npages_get(&shard->pac.ecache_muzzy);
}
void
pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
size_t *resident) {
cassert(config_stats);
pa_shard_stats_out->pac_stats.retained +=
ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE;
pa_shard_stats_out->edata_avail += atomic_load_zu(
&shard->edata_cache.count, ATOMIC_RELAXED);
size_t resident_pgs = 0;
resident_pgs += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
resident_pgs += ecache_npages_get(&shard->pac.ecache_dirty);
*resident += (resident_pgs << LG_PAGE);
/* Dirty decay stats */
locked_inc_u64_unsynchronized(
&pa_shard_stats_out->pac_stats.decay_dirty.npurge,
locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
&shard->pac.stats->decay_dirty.npurge));
locked_inc_u64_unsynchronized(
&pa_shard_stats_out->pac_stats.decay_dirty.nmadvise,
locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
&shard->pac.stats->decay_dirty.nmadvise));
locked_inc_u64_unsynchronized(
&pa_shard_stats_out->pac_stats.decay_dirty.purged,
locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
&shard->pac.stats->decay_dirty.purged));
/* Muzzy decay stats */
locked_inc_u64_unsynchronized(
&pa_shard_stats_out->pac_stats.decay_muzzy.npurge,
locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
&shard->pac.stats->decay_muzzy.npurge));
locked_inc_u64_unsynchronized(
&pa_shard_stats_out->pac_stats.decay_muzzy.nmadvise,
locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
&shard->pac.stats->decay_muzzy.nmadvise));
locked_inc_u64_unsynchronized(
&pa_shard_stats_out->pac_stats.decay_muzzy.purged,
locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
&shard->pac.stats->decay_muzzy.purged));
atomic_load_add_store_zu(&pa_shard_stats_out->pac_stats.abandoned_vm,
atomic_load_zu(&shard->pac.stats->abandoned_vm, ATOMIC_RELAXED));
for (pszind_t i = 0; i < SC_NPSIZES; i++) {
size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes,
retained_bytes;
dirty = ecache_nextents_get(&shard->pac.ecache_dirty, i);
muzzy = ecache_nextents_get(&shard->pac.ecache_muzzy, i);
retained = ecache_nextents_get(&shard->pac.ecache_retained, i);
dirty_bytes = ecache_nbytes_get(&shard->pac.ecache_dirty, i);
muzzy_bytes = ecache_nbytes_get(&shard->pac.ecache_muzzy, i);
retained_bytes = ecache_nbytes_get(&shard->pac.ecache_retained,
i);
estats_out[i].ndirty = dirty;
estats_out[i].nmuzzy = muzzy;
estats_out[i].nretained = retained;
estats_out[i].dirty_bytes = dirty_bytes;
estats_out[i].muzzy_bytes = muzzy_bytes;
estats_out[i].retained_bytes = retained_bytes;
}
if (shard->ever_used_hpa) {
hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out);
sec_stats_merge(tsdn, &shard->hpa_sec, sec_stats_out);
}
}
static void
pa_shard_mtx_stats_read_single(tsdn_t *tsdn, mutex_prof_data_t *mutex_prof_data,
malloc_mutex_t *mtx, int ind) {
malloc_mutex_lock(tsdn, mtx);
malloc_mutex_prof_read(tsdn, &mutex_prof_data[ind], mtx);
malloc_mutex_unlock(tsdn, mtx);
}
void
pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]) {
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->edata_cache.mtx, arena_prof_mutex_extent_avail);
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->pac.ecache_dirty.mtx, arena_prof_mutex_extents_dirty);
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->pac.ecache_muzzy.mtx, arena_prof_mutex_extents_muzzy);
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->pac.ecache_retained.mtx, arena_prof_mutex_extents_retained);
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->pac.decay_dirty.mtx, arena_prof_mutex_decay_dirty);
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy);
if (shard->ever_used_hpa) {
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->hpa_shard.mtx, arena_prof_mutex_hpa_shard);
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->hpa_shard.grow_mtx,
arena_prof_mutex_hpa_shard_grow);
sec_mutex_stats_read(tsdn, &shard->hpa_sec,
&mutex_prof_data[arena_prof_mutex_hpa_sec]);
}
}
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/pac.h"
#include "jemalloc/internal/san.h"
static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool *deferred_work_generated);
static void pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated);
static uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
static inline void
pac_decay_data_get(pac_t *pac, extent_state_t state,
decay_t **r_decay, pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) {
switch(state) {
case extent_state_dirty:
*r_decay = &pac->decay_dirty;
*r_decay_stats = &pac->stats->decay_dirty;
*r_ecache = &pac->ecache_dirty;
return;
case extent_state_muzzy:
*r_decay = &pac->decay_muzzy;
*r_decay_stats = &pac->stats->decay_muzzy;
*r_ecache = &pac->ecache_muzzy;
return;
default:
unreachable();
}
}
bool
pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
edata_cache_t *edata_cache, nstime_t *cur_time,
size_t pac_oversize_threshold, ssize_t dirty_decay_ms,
ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx) {
unsigned ind = base_ind_get(base);
/*
* Delay coalescing for dirty extents despite the disruptive effect on
* memory layout for best-fit extent allocation, since cached extents
* are likely to be reused soon after deallocation, and the cost of
* merging/splitting extents is non-trivial.
*/
if (ecache_init(tsdn, &pac->ecache_dirty, extent_state_dirty, ind,
/* delay_coalesce */ true)) {
return true;
}
/*
* Coalesce muzzy extents immediately, because operations on them are in
* the critical path much less often than for dirty extents.
*/
if (ecache_init(tsdn, &pac->ecache_muzzy, extent_state_muzzy, ind,
/* delay_coalesce */ false)) {
return true;
}
/*
* Coalesce retained extents immediately, in part because they will
* never be evicted (and therefore there's no opportunity for delayed
* coalescing), but also because operations on retained extents are not
* in the critical path.
*/
if (ecache_init(tsdn, &pac->ecache_retained, extent_state_retained,
ind, /* delay_coalesce */ false)) {
return true;
}
exp_grow_init(&pac->exp_grow);
if (malloc_mutex_init(&pac->grow_mtx, "extent_grow",
WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
return true;
}
atomic_store_zu(&pac->oversize_threshold, pac_oversize_threshold,
ATOMIC_RELAXED);
if (decay_init(&pac->decay_dirty, cur_time, dirty_decay_ms)) {
return true;
}
if (decay_init(&pac->decay_muzzy, cur_time, muzzy_decay_ms)) {
return true;
}
if (san_bump_alloc_init(&pac->sba)) {
return true;
}
pac->base = base;
pac->emap = emap;
pac->edata_cache = edata_cache;
pac->stats = pac_stats;
pac->stats_mtx = stats_mtx;
atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
pac->pai.alloc = &pac_alloc_impl;
pac->pai.alloc_batch = &pai_alloc_batch_default;
pac->pai.expand = &pac_expand_impl;
pac->pai.shrink = &pac_shrink_impl;
pac->pai.dalloc = &pac_dalloc_impl;
pac->pai.dalloc_batch = &pai_dalloc_batch_default;
pac->pai.time_until_deferred_work = &pac_time_until_deferred_work;
return false;
}
static inline bool
pac_may_have_muzzy(pac_t *pac) {
return pac_decay_ms_get(pac, extent_state_muzzy) != 0;
}
static edata_t *
pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
size_t alignment, bool zero, bool guarded) {
assert(!guarded || alignment <= PAGE);
edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
NULL, size, alignment, zero, guarded);
if (edata == NULL && pac_may_have_muzzy(pac)) {
edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
NULL, size, alignment, zero, guarded);
}
if (edata == NULL) {
edata = ecache_alloc_grow(tsdn, pac, ehooks,
&pac->ecache_retained, NULL, size, alignment, zero,
guarded);
if (config_stats && edata != NULL) {
atomic_fetch_add_zu(&pac->stats->pac_mapped, size,
ATOMIC_RELAXED);
}
}
return edata;
}
static edata_t *
pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
size_t alignment, bool zero, bool frequent_reuse) {
assert(alignment <= PAGE);
edata_t *edata;
if (san_bump_enabled() && frequent_reuse) {
edata = san_bump_alloc(tsdn, &pac->sba, pac, ehooks, size,
zero);
} else {
size_t size_with_guards = san_two_side_guarded_sz(size);
/* Alloc a non-guarded extent first.*/
edata = pac_alloc_real(tsdn, pac, ehooks, size_with_guards,
/* alignment */ PAGE, zero, /* guarded */ false);
if (edata != NULL) {
/* Add guards around it. */
assert(edata_size_get(edata) == size_with_guards);
san_guard_pages_two_sided(tsdn, ehooks, edata,
pac->emap, true);
}
}
assert(edata == NULL || (edata_guarded_get(edata) &&
edata_size_get(edata) == size));
return edata;
}
static edata_t *
pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated) {
pac_t *pac = (pac_t *)self;
ehooks_t *ehooks = pac_ehooks_get(pac);
edata_t *edata = NULL;
/*
* The condition is an optimization - not frequently reused guarded
* allocations are never put in the ecache. pac_alloc_real also
* doesn't grow retained for guarded allocations. So pac_alloc_real
* for such allocations would always return NULL.
* */
if (!guarded || frequent_reuse) {
edata = pac_alloc_real(tsdn, pac, ehooks, size, alignment,
zero, guarded);
}
if (edata == NULL && guarded) {
/* No cached guarded extents; creating a new one. */
edata = pac_alloc_new_guarded(tsdn, pac, ehooks, size,
alignment, zero, frequent_reuse);
}
return edata;
}
static bool
pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool zero, bool *deferred_work_generated) {
pac_t *pac = (pac_t *)self;
ehooks_t *ehooks = pac_ehooks_get(pac);
size_t mapped_add = 0;
size_t expand_amount = new_size - old_size;
if (ehooks_merge_will_fail(ehooks)) {
return true;
}
edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
edata, expand_amount, PAGE, zero, /* guarded*/ false);
if (trail == NULL) {
trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
edata, expand_amount, PAGE, zero, /* guarded*/ false);
}
if (trail == NULL) {
trail = ecache_alloc_grow(tsdn, pac, ehooks,
&pac->ecache_retained, edata, expand_amount, PAGE, zero,
/* guarded */ false);
mapped_add = expand_amount;
}
if (trail == NULL) {
return true;
}
if (extent_merge_wrapper(tsdn, pac, ehooks, edata, trail)) {
extent_dalloc_wrapper(tsdn, pac, ehooks, trail);
return true;
}
if (config_stats && mapped_add > 0) {
atomic_fetch_add_zu(&pac->stats->pac_mapped, mapped_add,
ATOMIC_RELAXED);
}
return false;
}
static bool
pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool *deferred_work_generated) {
pac_t *pac = (pac_t *)self;
ehooks_t *ehooks = pac_ehooks_get(pac);
size_t shrink_amount = old_size - new_size;
if (ehooks_split_will_fail(ehooks)) {
return true;
}
edata_t *trail = extent_split_wrapper(tsdn, pac, ehooks, edata,
new_size, shrink_amount, /* holding_core_locks */ false);
if (trail == NULL) {
return true;
}
ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, trail);
*deferred_work_generated = true;
return false;
}
static void
pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated) {
pac_t *pac = (pac_t *)self;
ehooks_t *ehooks = pac_ehooks_get(pac);
if (edata_guarded_get(edata)) {
/*
* Because cached guarded extents do exact fit only, large
* guarded extents are restored on dalloc eagerly (otherwise
* they will not be reused efficiently). Slab sizes have a
* limited number of size classes, and tend to cycle faster.
*
* In the case where coalesce is restrained (VirtualFree on
* Windows), guarded extents are also not cached -- otherwise
* during arena destroy / reset, the retained extents would not
* be whole regions (i.e. they are split between regular and
* guarded).
*/
if (!edata_slab_get(edata) || !maps_coalesce) {
assert(edata_size_get(edata) >= SC_LARGE_MINCLASS ||
!maps_coalesce);
san_unguard_pages_two_sided(tsdn, ehooks, edata,
pac->emap);
}
}
ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata);
/* Purging of deallocated pages is deferred */
*deferred_work_generated = true;
}
static inline uint64_t
pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) {
if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
/* Use minimal interval if decay is contended. */
return BACKGROUND_THREAD_DEFERRED_MIN;
}
uint64_t result = decay_ns_until_purge(decay, npages,
ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD);
malloc_mutex_unlock(tsdn, &decay->mtx);
return result;
}
static uint64_t
pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
uint64_t time;
pac_t *pac = (pac_t *)self;
time = pac_ns_until_purge(tsdn,
&pac->decay_dirty,
ecache_npages_get(&pac->ecache_dirty));
if (time == BACKGROUND_THREAD_DEFERRED_MIN) {
return time;
}
uint64_t muzzy = pac_ns_until_purge(tsdn,
&pac->decay_muzzy,
ecache_npages_get(&pac->ecache_muzzy));
if (muzzy < time) {
time = muzzy;
}
return time;
}
bool
pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
size_t *new_limit) {
pszind_t new_ind JEMALLOC_CC_SILENCE_INIT(0);
if (new_limit != NULL) {
size_t limit = *new_limit;
/* Grow no more than the new limit. */
if ((new_ind = sz_psz2ind(limit + 1) - 1) >= SC_NPSIZES) {
return true;
}
}
malloc_mutex_lock(tsdn, &pac->grow_mtx);
if (old_limit != NULL) {
*old_limit = sz_pind2sz(pac->exp_grow.limit);
}
if (new_limit != NULL) {
pac->exp_grow.limit = new_ind;
}
malloc_mutex_unlock(tsdn, &pac->grow_mtx);
return false;
}
static size_t
pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
size_t npages_limit, size_t npages_decay_max,
edata_list_inactive_t *result) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
ehooks_t *ehooks = pac_ehooks_get(pac);
/* Stash extents according to npages_limit. */
size_t nstashed = 0;
while (nstashed < npages_decay_max) {
edata_t *edata = ecache_evict(tsdn, pac, ehooks, ecache,
npages_limit);
if (edata == NULL) {
break;
}
edata_list_inactive_append(result, edata);
nstashed += edata_size_get(edata) >> LG_PAGE;
}
return nstashed;
}
static size_t
pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
edata_list_inactive_t *decay_extents) {
bool err;
size_t nmadvise = 0;
size_t nunmapped = 0;
size_t npurged = 0;
ehooks_t *ehooks = pac_ehooks_get(pac);
bool try_muzzy = !fully_decay
&& pac_decay_ms_get(pac, extent_state_muzzy) != 0;
for (edata_t *edata = edata_list_inactive_first(decay_extents); edata !=
NULL; edata = edata_list_inactive_first(decay_extents)) {
edata_list_inactive_remove(decay_extents, edata);
size_t size = edata_size_get(edata);
size_t npages = size >> LG_PAGE;
nmadvise++;
npurged += npages;
switch (ecache->state) {
case extent_state_active:
not_reached();
case extent_state_dirty:
if (try_muzzy) {
err = extent_purge_lazy_wrapper(tsdn, ehooks,
edata, /* offset */ 0, size);
if (!err) {
ecache_dalloc(tsdn, pac, ehooks,
&pac->ecache_muzzy, edata);
break;
}
}
JEMALLOC_FALLTHROUGH;
case extent_state_muzzy:
extent_dalloc_wrapper(tsdn, pac, ehooks, edata);
nunmapped += npages;
break;
case extent_state_retained:
default:
not_reached();
}
}
if (config_stats) {
LOCKEDINT_MTX_LOCK(tsdn, *pac->stats_mtx);
locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx),
&decay_stats->npurge, 1);
locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx),
&decay_stats->nmadvise, nmadvise);
locked_inc_u64(tsdn, LOCKEDINT_MTX(*pac->stats_mtx),
&decay_stats->purged, npurged);
LOCKEDINT_MTX_UNLOCK(tsdn, *pac->stats_mtx);
atomic_fetch_sub_zu(&pac->stats->pac_mapped,
nunmapped << LG_PAGE, ATOMIC_RELAXED);
}
return npurged;
}
/*
* npages_limit: Decay at most npages_decay_max pages without violating the
* invariant: (ecache_npages_get(ecache) >= npages_limit). We need an upper
* bound on number of pages in order to prevent unbounded growth (namely in
* stashed), otherwise unbounded new pages could be added to extents during the
* current decay run, so that the purging thread never finishes.
*/
static void
pac_decay_to_limit(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
size_t npages_limit, size_t npages_decay_max) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 1);
if (decay->purging || npages_decay_max == 0) {
return;
}
decay->purging = true;
malloc_mutex_unlock(tsdn, &decay->mtx);
edata_list_inactive_t decay_extents;
edata_list_inactive_init(&decay_extents);
size_t npurge = pac_stash_decayed(tsdn, pac, ecache, npages_limit,
npages_decay_max, &decay_extents);
if (npurge != 0) {
size_t npurged = pac_decay_stashed(tsdn, pac, decay,
decay_stats, ecache, fully_decay, &decay_extents);
assert(npurged == npurge);
}
malloc_mutex_lock(tsdn, &decay->mtx);
decay->purging = false;
}
void
pac_decay_all(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay) {
malloc_mutex_assert_owner(tsdn, &decay->mtx);
pac_decay_to_limit(tsdn, pac, decay, decay_stats, ecache, fully_decay,
/* npages_limit */ 0, ecache_npages_get(ecache));
}
static void
pac_decay_try_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
pac_decay_stats_t *decay_stats, ecache_t *ecache,
size_t current_npages, size_t npages_limit) {
if (current_npages > npages_limit) {
pac_decay_to_limit(tsdn, pac, decay, decay_stats, ecache,
/* fully_decay */ false, npages_limit,
current_npages - npages_limit);
}
}
bool
pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
pac_decay_stats_t *decay_stats, ecache_t *ecache,
pac_purge_eagerness_t eagerness) {
malloc_mutex_assert_owner(tsdn, &decay->mtx);
/* Purge all or nothing if the option is disabled. */
ssize_t decay_ms = decay_ms_read(decay);
if (decay_ms <= 0) {
if (decay_ms == 0) {
pac_decay_to_limit(tsdn, pac, decay, decay_stats,
ecache, /* fully_decay */ false,
/* npages_limit */ 0, ecache_npages_get(ecache));
}
return false;
}
/*
* If the deadline has been reached, advance to the current epoch and
* purge to the new limit if necessary. Note that dirty pages created
* during the current epoch are not subject to purge until a future
* epoch, so as a result purging only happens during epoch advances, or
* being triggered by background threads (scheduled event).
*/
nstime_t time;
nstime_init_update(&time);
size_t npages_current = ecache_npages_get(ecache);
bool epoch_advanced = decay_maybe_advance_epoch(decay, &time,
npages_current);
if (eagerness == PAC_PURGE_ALWAYS
|| (epoch_advanced && eagerness == PAC_PURGE_ON_EPOCH_ADVANCE)) {
size_t npages_limit = decay_npages_limit_get(decay);
pac_decay_try_purge(tsdn, pac, decay, decay_stats, ecache,
npages_current, npages_limit);
}
return epoch_advanced;
}
bool
pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
ssize_t decay_ms, pac_purge_eagerness_t eagerness) {
decay_t *decay;
pac_decay_stats_t *decay_stats;
ecache_t *ecache;
pac_decay_data_get(pac, state, &decay, &decay_stats, &ecache);
if (!decay_ms_valid(decay_ms)) {
return true;
}
malloc_mutex_lock(tsdn, &decay->mtx);
/*
* Restart decay backlog from scratch, which may cause many dirty pages
* to be immediately purged. It would conceptually be possible to map
* the old backlog onto the new backlog, but there is no justification
* for such complexity since decay_ms changes are intended to be
* infrequent, either between the {-1, 0, >0} states, or a one-time
* arbitrary change during initial arena configuration.
*/
nstime_t cur_time;
nstime_init_update(&cur_time);
decay_reinit(decay, &cur_time, decay_ms);
pac_maybe_decay_purge(tsdn, pac, decay, decay_stats, ecache, eagerness);
malloc_mutex_unlock(tsdn, &decay->mtx);
return false;
}
ssize_t
pac_decay_ms_get(pac_t *pac, extent_state_t state) {
decay_t *decay;
pac_decay_stats_t *decay_stats;
ecache_t *ecache;
pac_decay_data_get(pac, state, &decay, &decay_stats, &ecache);
return decay_ms_read(decay);
}
void
pac_reset(tsdn_t *tsdn, pac_t *pac) {
/*
* No-op for now; purging is still done at the arena-level. It should
* get moved in here, though.
*/
(void)tsdn;
(void)pac;
}
void
pac_destroy(tsdn_t *tsdn, pac_t *pac) {
assert(ecache_npages_get(&pac->ecache_dirty) == 0);
assert(ecache_npages_get(&pac->ecache_muzzy) == 0);
/*
* Iterate over the retained extents and destroy them. This gives the
* extent allocator underlying the extent hooks an opportunity to unmap
* all retained memory without having to keep its own metadata
* structures. In practice, virtual memory for dss-allocated extents is
* leaked here, so best practice is to avoid dss for arenas to be
* destroyed, or provide custom extent hooks that track retained
* dss-based extents for later reuse.
*/
ehooks_t *ehooks = pac_ehooks_get(pac);
edata_t *edata;
while ((edata = ecache_evict(tsdn, pac, ehooks,
&pac->ecache_retained, 0)) != NULL) {
extent_destroy_wrapper(tsdn, pac, ehooks, edata);
}
}
#define JEMALLOC_PAGES_C_
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/pages.h"
......@@ -14,6 +13,14 @@
#include <vm/vm_param.h>
#endif
#endif
#ifdef __NetBSD__
#include <sys/bitops.h> /* ilog2 */
#endif
#ifdef JEMALLOC_HAVE_VM_MAKE_TAG
#define PAGES_FD_TAG VM_MAKE_TAG(101U)
#else
#define PAGES_FD_TAG -1
#endif
/******************************************************************************/
/* Data. */
......@@ -40,6 +47,57 @@ thp_mode_t init_system_thp_mode;
/* Runtime support for lazy purge. Irrelevant when !pages_can_purge_lazy. */
static bool pages_can_purge_lazy_runtime = true;
#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
static int madvise_dont_need_zeros_is_faulty = -1;
/**
* Check that MADV_DONTNEED will actually zero pages on subsequent access.
*
* Since qemu does not support this, yet [1], and you can get very tricky
* assert if you will run program with jemalloc in use under qemu:
*
* <jemalloc>: ../contrib/jemalloc/src/extent.c:1195: Failed assertion: "p[i] == 0"
*
* [1]: https://patchwork.kernel.org/patch/10576637/
*/
static int madvise_MADV_DONTNEED_zeroes_pages()
{
int works = -1;
size_t size = PAGE;
void * addr = mmap(NULL, size, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
if (addr == MAP_FAILED) {
malloc_write("<jemalloc>: Cannot allocate memory for "
"MADV_DONTNEED check\n");
if (opt_abort) {
abort();
}
}
memset(addr, 'A', size);
if (madvise(addr, size, MADV_DONTNEED) == 0) {
works = memchr(addr, 'A', size) == NULL;
} else {
/*
* If madvise() does not support MADV_DONTNEED, then we can
* call it anyway, and use it's return code.
*/
works = 1;
}
if (munmap(addr, size) != 0) {
malloc_write("<jemalloc>: Cannot deallocate memory for "
"MADV_DONTNEED check\n");
if (opt_abort) {
abort();
}
}
return works;
}
#endif
/******************************************************************************/
/*
* Function prototypes for static functions that are referenced prior to
......@@ -74,9 +132,21 @@ os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
* of existing mappings, and we only want to create new mappings.
*/
{
#ifdef __NetBSD__
/*
* On NetBSD PAGE for a platform is defined to the
* maximum page size of all machine architectures
* for that platform, so that we can use the same
* binaries across all machine architectures.
*/
if (alignment > os_page || PAGE > os_page) {
unsigned int a = ilog2(MAX(alignment, PAGE));
mmap_flags |= MAP_ALIGNED(a);
}
#endif
int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
ret = mmap(addr, size, prot, mmap_flags, -1, 0);
ret = mmap(addr, size, prot, mmap_flags, PAGES_FD_TAG, 0);
}
assert(ret != NULL);
......@@ -197,8 +267,8 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
flags |= MAP_FIXED | MAP_EXCL;
} else {
unsigned alignment_bits = ffs_zu(alignment);
assert(alignment_bits > 1);
flags |= MAP_ALIGNED(alignment_bits - 1);
assert(alignment_bits > 0);
flags |= MAP_ALIGNED(alignment_bits);
}
void *ret = mmap(addr, size, prot, flags, -1, 0);
......@@ -246,14 +316,10 @@ pages_unmap(void *addr, size_t size) {
}
static bool
pages_commit_impl(void *addr, size_t size, bool commit) {
os_pages_commit(void *addr, size_t size, bool commit) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
if (os_overcommits) {
return true;
}
#ifdef _WIN32
return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,
PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT)));
......@@ -261,7 +327,7 @@ pages_commit_impl(void *addr, size_t size, bool commit) {
{
int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED,
-1, 0);
PAGES_FD_TAG, 0);
if (result == MAP_FAILED) {
return true;
}
......@@ -278,6 +344,15 @@ pages_commit_impl(void *addr, size_t size, bool commit) {
#endif
}
static bool
pages_commit_impl(void *addr, size_t size, bool commit) {
if (os_overcommits) {
return true;
}
return os_pages_commit(addr, size, commit);
}
bool
pages_commit(void *addr, size_t size) {
return pages_commit_impl(addr, size, true);
......@@ -288,6 +363,66 @@ pages_decommit(void *addr, size_t size) {
return pages_commit_impl(addr, size, false);
}
void
pages_mark_guards(void *head, void *tail) {
assert(head != NULL || tail != NULL);
assert(head == NULL || tail == NULL ||
(uintptr_t)head < (uintptr_t)tail);
#ifdef JEMALLOC_HAVE_MPROTECT
if (head != NULL) {
mprotect(head, PAGE, PROT_NONE);
}
if (tail != NULL) {
mprotect(tail, PAGE, PROT_NONE);
}
#else
/* Decommit sets to PROT_NONE / MEM_DECOMMIT. */
if (head != NULL) {
os_pages_commit(head, PAGE, false);
}
if (tail != NULL) {
os_pages_commit(tail, PAGE, false);
}
#endif
}
void
pages_unmark_guards(void *head, void *tail) {
assert(head != NULL || tail != NULL);
assert(head == NULL || tail == NULL ||
(uintptr_t)head < (uintptr_t)tail);
#ifdef JEMALLOC_HAVE_MPROTECT
bool head_and_tail = (head != NULL) && (tail != NULL);
size_t range = head_and_tail ?
(uintptr_t)tail - (uintptr_t)head + PAGE :
SIZE_T_MAX;
/*
* The amount of work that the kernel does in mprotect depends on the
* range argument. SC_LARGE_MINCLASS is an arbitrary threshold chosen
* to prevent kernel from doing too much work that would outweigh the
* savings of performing one less system call.
*/
bool ranged_mprotect = head_and_tail && range <= SC_LARGE_MINCLASS;
if (ranged_mprotect) {
mprotect(head, range, PROT_READ | PROT_WRITE);
} else {
if (head != NULL) {
mprotect(head, PAGE, PROT_READ | PROT_WRITE);
}
if (tail != NULL) {
mprotect(tail, PAGE, PROT_READ | PROT_WRITE);
}
}
#else
if (head != NULL) {
os_pages_commit(head, PAGE, true);
}
if (tail != NULL) {
os_pages_commit(tail, PAGE, true);
}
#endif
}
bool
pages_purge_lazy(void *addr, size_t size) {
assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
......@@ -318,6 +453,9 @@ pages_purge_lazy(void *addr, size_t size) {
#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \
!defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)
return (madvise(addr, size, MADV_DONTNEED) != 0);
#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) && \
!defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS)
return (posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0);
#else
not_reached();
#endif
......@@ -334,7 +472,12 @@ pages_purge_forced(void *addr, size_t size) {
#if defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \
defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)
return (madvise(addr, size, MADV_DONTNEED) != 0);
return (unlikely(madvise_dont_need_zeros_is_faulty) ||
madvise(addr, size, MADV_DONTNEED) != 0);
#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) && \
defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS)
return (unlikely(madvise_dont_need_zeros_is_faulty) ||
posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0);
#elif defined(JEMALLOC_MAPS_COALESCE)
/* Try to overlay a new demand-zeroed mapping. */
return pages_commit(addr, size);
......@@ -349,8 +492,13 @@ pages_huge_impl(void *addr, size_t size, bool aligned) {
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
assert(HUGEPAGE_CEILING(size) == size);
}
#ifdef JEMALLOC_HAVE_MADVISE_HUGE
#if defined(JEMALLOC_HAVE_MADVISE_HUGE)
return (madvise(addr, size, MADV_HUGEPAGE) != 0);
#elif defined(JEMALLOC_HAVE_MEMCNTL)
struct memcntl_mha m = {0};
m.mha_cmd = MHA_MAPSIZE_VA;
m.mha_pagesize = HUGEPAGE;
return (memcntl(addr, size, MC_HAT_ADVISE, (caddr_t)&m, 0, 0) == 0);
#else
return true;
#endif
......@@ -394,8 +542,10 @@ bool
pages_dontdump(void *addr, size_t size) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
#ifdef JEMALLOC_MADVISE_DONTDUMP
#if defined(JEMALLOC_MADVISE_DONTDUMP)
return madvise(addr, size, MADV_DONTDUMP) != 0;
#elif defined(JEMALLOC_MADVISE_NOCORE)
return madvise(addr, size, MADV_NOCORE) != 0;
#else
return false;
#endif
......@@ -405,8 +555,10 @@ bool
pages_dodump(void *addr, size_t size) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
#ifdef JEMALLOC_MADVISE_DONTDUMP
#if defined(JEMALLOC_MADVISE_DONTDUMP)
return madvise(addr, size, MADV_DODUMP) != 0;
#elif defined(JEMALLOC_MADVISE_NOCORE)
return madvise(addr, size, MADV_CORE) != 0;
#else
return false;
#endif
......@@ -547,14 +699,14 @@ pages_set_thp_state (void *ptr, size_t size) {
static void
init_thp_state(void) {
if (!have_madvise_huge) {
if (!have_madvise_huge && !have_memcntl) {
if (metadata_thp_enabled() && opt_abort) {
malloc_write("<jemalloc>: no MADV_HUGEPAGE support\n");
abort();
}
goto label_error;
}
#if defined(JEMALLOC_HAVE_MADVISE_HUGE)
static const char sys_state_madvise[] = "always [madvise] never\n";
static const char sys_state_always[] = "[always] madvise never\n";
static const char sys_state_never[] = "always madvise [never]\n";
......@@ -563,6 +715,9 @@ init_thp_state(void) {
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
int fd = (int)syscall(SYS_open,
"/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
#elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat)
int fd = (int)syscall(SYS_openat,
AT_FDCWD, "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
#else
int fd = open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
#endif
......@@ -578,7 +733,7 @@ init_thp_state(void) {
#endif
if (nread < 0) {
goto label_error;
goto label_error;
}
if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) {
......@@ -591,6 +746,10 @@ init_thp_state(void) {
goto label_error;
}
return;
#elif defined(JEMALLOC_HAVE_MEMCNTL)
init_system_thp_mode = thp_mode_default;
return;
#endif
label_error:
opt_thp = init_system_thp_mode = thp_mode_not_supported;
}
......@@ -606,6 +765,20 @@ pages_boot(void) {
return true;
}
#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
if (!opt_trust_madvise) {
madvise_dont_need_zeros_is_faulty = !madvise_MADV_DONTNEED_zeroes_pages();
if (madvise_dont_need_zeros_is_faulty) {
malloc_write("<jemalloc>: MADV_DONTNEED does not work (memset will be used instead)\n");
malloc_write("<jemalloc>: (This is the expected behaviour if you are running under QEMU)\n");
}
} else {
/* In case opt_trust_madvise is disable,
* do not do runtime check */
madvise_dont_need_zeros_is_faulty = 0;
}
#endif
#ifndef _WIN32
mmap_flags = MAP_PRIVATE | MAP_ANON;
#endif
......@@ -619,6 +792,8 @@ pages_boot(void) {
mmap_flags |= MAP_NORESERVE;
}
# endif
#elif defined(__NetBSD__)
os_overcommits = true;
#else
os_overcommits = false;
#endif
......
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
size_t
pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
edata_list_active_t *results, bool *deferred_work_generated) {
for (size_t i = 0; i < nallocs; i++) {
bool deferred_by_alloc = false;
edata_t *edata = pai_alloc(tsdn, self, size, PAGE,
/* zero */ false, /* guarded */ false,
/* frequent_reuse */ false, &deferred_by_alloc);
*deferred_work_generated |= deferred_by_alloc;
if (edata == NULL) {
return i;
}
edata_list_active_append(results, edata);
}
return nallocs;
}
void
pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self,
edata_list_active_t *list, bool *deferred_work_generated) {
edata_t *edata;
while ((edata = edata_list_active_first(list)) != NULL) {
bool deferred_by_dalloc = false;
edata_list_active_remove(list, edata);
pai_dalloc(tsdn, self, edata, &deferred_by_dalloc);
*deferred_work_generated |= deferred_by_dalloc;
}
}
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/peak_event.h"
#include "jemalloc/internal/activity_callback.h"
#include "jemalloc/internal/peak.h"
/*
* Update every 64K by default. We're not exposing this as a configuration
* option for now; we don't want to bind ourselves too tightly to any particular
* performance requirements for small values, or guarantee that we'll even be
* able to provide fine-grained accuracy.
*/
#define PEAK_EVENT_WAIT (64 * 1024)
/* Update the peak with current tsd state. */
void
peak_event_update(tsd_t *tsd) {
uint64_t alloc = tsd_thread_allocated_get(tsd);
uint64_t dalloc = tsd_thread_deallocated_get(tsd);
peak_t *peak = tsd_peakp_get(tsd);
peak_update(peak, alloc, dalloc);
}
static void
peak_event_activity_callback(tsd_t *tsd) {
activity_callback_thunk_t *thunk = tsd_activity_callback_thunkp_get(
tsd);
uint64_t alloc = tsd_thread_allocated_get(tsd);
uint64_t dalloc = tsd_thread_deallocated_get(tsd);
if (thunk->callback != NULL) {
thunk->callback(thunk->uctx, alloc, dalloc);
}
}
/* Set current state to zero. */
void
peak_event_zero(tsd_t *tsd) {
uint64_t alloc = tsd_thread_allocated_get(tsd);
uint64_t dalloc = tsd_thread_deallocated_get(tsd);
peak_t *peak = tsd_peakp_get(tsd);
peak_set_zero(peak, alloc, dalloc);
}
uint64_t
peak_event_max(tsd_t *tsd) {
peak_t *peak = tsd_peakp_get(tsd);
return peak_max(peak);
}
uint64_t
peak_alloc_new_event_wait(tsd_t *tsd) {
return PEAK_EVENT_WAIT;
}
uint64_t
peak_alloc_postponed_event_wait(tsd_t *tsd) {
return TE_MIN_START_WAIT;
}
void
peak_alloc_event_handler(tsd_t *tsd, uint64_t elapsed) {
peak_event_update(tsd);
peak_event_activity_callback(tsd);
}
uint64_t
peak_dalloc_new_event_wait(tsd_t *tsd) {
return PEAK_EVENT_WAIT;
}
uint64_t
peak_dalloc_postponed_event_wait(tsd_t *tsd) {
return TE_MIN_START_WAIT;
}
void
peak_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed) {
peak_event_update(tsd);
peak_event_activity_callback(tsd);
}
#define JEMALLOC_PROF_C_
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/ckh.h"
#include "jemalloc/internal/hash.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/emitter.h"
#include "jemalloc/internal/counter.h"
#include "jemalloc/internal/prof_data.h"
#include "jemalloc/internal/prof_log.h"
#include "jemalloc/internal/prof_recent.h"
#include "jemalloc/internal/prof_stats.h"
#include "jemalloc/internal/prof_sys.h"
#include "jemalloc/internal/prof_hook.h"
#include "jemalloc/internal/thread_event.h"
/******************************************************************************/
#ifdef JEMALLOC_PROF_LIBUNWIND
#define UNW_LOCAL_ONLY
#include <libunwind.h>
#endif
#ifdef JEMALLOC_PROF_LIBGCC
/*
* We have a circular dependency -- jemalloc_internal.h tells us if we should
* use libgcc's unwinding functionality, but after we've included that, we've
* already hooked _Unwind_Backtrace. We'll temporarily disable hooking.
* This file implements the profiling "APIs" needed by other parts of jemalloc,
* and also manages the relevant "operational" data, mainly options and mutexes;
* the core profiling data structures are encapsulated in prof_data.c.
*/
#undef _Unwind_Backtrace
#include <unwind.h>
#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook)
#endif
/******************************************************************************/
/* Data. */
bool opt_prof = false;
bool opt_prof_active = true;
bool opt_prof_thread_active_init = true;
size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
bool opt_prof_gdump = false;
bool opt_prof_final = false;
bool opt_prof_leak = false;
bool opt_prof_accum = false;
bool opt_prof_log = false;
char opt_prof_prefix[
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
PATH_MAX +
#endif
1];
bool opt_prof = false;
bool opt_prof_active = true;
bool opt_prof_thread_active_init = true;
size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
bool opt_prof_gdump = false;
bool opt_prof_final = false;
bool opt_prof_leak = false;
bool opt_prof_leak_error = false;
bool opt_prof_accum = false;
char opt_prof_prefix[PROF_DUMP_FILENAME_LEN];
bool opt_prof_sys_thread_name = false;
bool opt_prof_unbias = true;
/* Accessed via prof_sample_event_handler(). */
static counter_accum_t prof_idump_accumulated;
/*
* Initialized as opt_prof_active, and accessed via
* prof_active_[gs]et{_unlocked,}().
*/
bool prof_active;
static malloc_mutex_t prof_active_mtx;
bool prof_active_state;
static malloc_mutex_t prof_active_mtx;
/*
* Initialized as opt_prof_thread_active_init, and accessed via
* prof_thread_active_init_[gs]et().
*/
static bool prof_thread_active_init;
static malloc_mutex_t prof_thread_active_init_mtx;
static bool prof_thread_active_init;
static malloc_mutex_t prof_thread_active_init_mtx;
/*
* Initialized as opt_prof_gdump, and accessed via
* prof_gdump_[gs]et{_unlocked,}().
*/
bool prof_gdump_val;
static malloc_mutex_t prof_gdump_mtx;
uint64_t prof_interval = 0;
size_t lg_prof_sample;
typedef enum prof_logging_state_e prof_logging_state_t;
enum prof_logging_state_e {
prof_logging_state_stopped,
prof_logging_state_started,
prof_logging_state_dumping
};
/*
* - stopped: log_start never called, or previous log_stop has completed.
* - started: log_start called, log_stop not called yet. Allocations are logged.
* - dumping: log_stop called but not finished; samples are not logged anymore.
*/
prof_logging_state_t prof_logging_state = prof_logging_state_stopped;
#ifdef JEMALLOC_JET
static bool prof_log_dummy = false;
#endif
/* Incremented for every log file that is output. */
static uint64_t log_seq = 0;
static char log_filename[
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
PATH_MAX +
#endif
1];
/* Timestamp for most recent call to log_start(). */
static nstime_t log_start_timestamp = NSTIME_ZERO_INITIALIZER;
/* Increment these when adding to the log_bt and log_thr linked lists. */
static size_t log_bt_index = 0;
static size_t log_thr_index = 0;
/* Linked list node definitions. These are only used in prof.c. */
typedef struct prof_bt_node_s prof_bt_node_t;
struct prof_bt_node_s {
prof_bt_node_t *next;
size_t index;
prof_bt_t bt;
/* Variable size backtrace vector pointed to by bt. */
void *vec[1];
};
typedef struct prof_thr_node_s prof_thr_node_t;
struct prof_thr_node_s {
prof_thr_node_t *next;
size_t index;
uint64_t thr_uid;
/* Variable size based on thr_name_sz. */
char name[1];
};
typedef struct prof_alloc_node_s prof_alloc_node_t;
/* This is output when logging sampled allocations. */
struct prof_alloc_node_s {
prof_alloc_node_t *next;
/* Indices into an array of thread data. */
size_t alloc_thr_ind;
size_t free_thr_ind;
/* Indices into an array of backtraces. */
size_t alloc_bt_ind;
size_t free_bt_ind;
uint64_t alloc_time_ns;
uint64_t free_time_ns;
size_t usize;
};
/*
* Created on the first call to prof_log_start and deleted on prof_log_stop.
* These are the backtraces and threads that have already been logged by an
* allocation.
*/
static bool log_tables_initialized = false;
static ckh_t log_bt_node_set;
static ckh_t log_thr_node_set;
/* Store linked lists for logged data. */
static prof_bt_node_t *log_bt_first = NULL;
static prof_bt_node_t *log_bt_last = NULL;
static prof_thr_node_t *log_thr_first = NULL;
static prof_thr_node_t *log_thr_last = NULL;
static prof_alloc_node_t *log_alloc_first = NULL;
static prof_alloc_node_t *log_alloc_last = NULL;
/* Protects the prof_logging_state and any log_{...} variable. */
static malloc_mutex_t log_mtx;
/*
* Table of mutexes that are shared among gctx's. These are leaf locks, so
* there is no problem with using them for more than one gctx at the same time.
* The primary motivation for this sharing though is that gctx's are ephemeral,
* and destroying mutexes causes complications for systems that allocate when
* creating/destroying mutexes.
*/
static malloc_mutex_t *gctx_locks;
static atomic_u_t cum_gctxs; /* Atomic counter. */
/*
* Table of mutexes that are shared among tdata's. No operations require
* holding multiple tdata locks, so there is no problem with using them for more
* than one tdata at the same time, even though a gctx lock may be acquired
* while holding a tdata lock.
*/
static malloc_mutex_t *tdata_locks;
/*
* Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data
* structure that knows about all backtraces currently captured.
*/
static ckh_t bt2gctx;
/* Non static to enable profiling. */
malloc_mutex_t bt2gctx_mtx;
/*
* Tree of all extant prof_tdata_t structures, regardless of state,
* {attached,detached,expired}.
*/
static prof_tdata_tree_t tdatas;
static malloc_mutex_t tdatas_mtx;
bool prof_gdump_val;
static malloc_mutex_t prof_gdump_mtx;
static uint64_t next_thr_uid;
static malloc_mutex_t next_thr_uid_mtx;
uint64_t prof_interval = 0;
static malloc_mutex_t prof_dump_seq_mtx;
static uint64_t prof_dump_seq;
static uint64_t prof_dump_iseq;
static uint64_t prof_dump_mseq;
static uint64_t prof_dump_useq;
size_t lg_prof_sample;
/*
* This buffer is rather large for stack allocation, so use a single buffer for
* all profile dumps.
*/
static malloc_mutex_t prof_dump_mtx;
static char prof_dump_buf[
/* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
PROF_DUMP_BUFSIZE
#else
1
#endif
];
static size_t prof_dump_buf_end;
static int prof_dump_fd;
static uint64_t next_thr_uid;
static malloc_mutex_t next_thr_uid_mtx;
/* Do not dump any profiles until bootstrapping is complete. */
static bool prof_booted = false;
/******************************************************************************/
/*
* Function prototypes for static functions that are referenced prior to
* definition.
*/
static bool prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx);
static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
static bool prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
bool even_if_attached);
static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
bool even_if_attached);
static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name);
/* Hashtable functions for log_bt_node_set and log_thr_node_set. */
static void prof_thr_node_hash(const void *key, size_t r_hash[2]);
static bool prof_thr_node_keycomp(const void *k1, const void *k2);
static void prof_bt_node_hash(const void *key, size_t r_hash[2]);
static bool prof_bt_node_keycomp(const void *k1, const void *k2);
/******************************************************************************/
/* Red-black trees. */
static int
prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) {
uint64_t a_thr_uid = a->thr_uid;
uint64_t b_thr_uid = b->thr_uid;
int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
if (ret == 0) {
uint64_t a_thr_discrim = a->thr_discrim;
uint64_t b_thr_discrim = b->thr_discrim;
ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
b_thr_discrim);
if (ret == 0) {
uint64_t a_tctx_uid = a->tctx_uid;
uint64_t b_tctx_uid = b->tctx_uid;
ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
b_tctx_uid);
}
}
return ret;
}
rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
tctx_link, prof_tctx_comp)
static int
prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) {
unsigned a_len = a->bt.len;
unsigned b_len = b->bt.len;
unsigned comp_len = (a_len < b_len) ? a_len : b_len;
int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
if (ret == 0) {
ret = (a_len > b_len) - (a_len < b_len);
}
return ret;
}
bool prof_booted = false;
rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
prof_gctx_comp)
static int
prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) {
int ret;
uint64_t a_uid = a->thr_uid;
uint64_t b_uid = b->thr_uid;
ret = ((a_uid > b_uid) - (a_uid < b_uid));
if (ret == 0) {
uint64_t a_discrim = a->thr_discrim;
uint64_t b_discrim = b->thr_discrim;
ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
}
return ret;
}
/* Logically a prof_backtrace_hook_t. */
atomic_p_t prof_backtrace_hook;
rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
prof_tdata_comp)
/* Logically a prof_dump_hook_t. */
atomic_p_t prof_dump_hook;
/******************************************************************************/
void
prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) {
prof_tdata_t *tdata;
prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx) {
cassert(config_prof);
if (updated) {
/*
* Compute a new sample threshold. This isn't very important in
* practice, because this function is rarely executed, so the
* potential for sample bias is minimal except in contrived
* programs.
*/
tdata = prof_tdata_get(tsd, true);
if (tdata != NULL) {
prof_sample_threshold_update(tdata);
}
if (tsd_reentrancy_level_get(tsd) > 0) {
assert((uintptr_t)tctx == (uintptr_t)1U);
return;
}
if ((uintptr_t)tctx > (uintptr_t)1U) {
malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
tctx->prepared = false;
if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) {
prof_tctx_destroy(tsd, tctx);
} else {
malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
}
prof_tctx_try_destroy(tsd, tctx);
}
}
void
prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
prof_tctx_t *tctx) {
prof_tctx_set(tsdn, ptr, usize, NULL, tctx);
prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t size,
size_t usize, prof_tctx_t *tctx) {
cassert(config_prof);
if (opt_prof_sys_thread_name) {
prof_sys_thread_name_fetch(tsd);
}
/* Get the current time and set this in the extent_t. We'll read this
* when free() is called. */
nstime_t t = NSTIME_ZERO_INITIALIZER;
nstime_update(&t);
prof_alloc_time_set(tsdn, ptr, NULL, t);
edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global,
ptr);
prof_info_set(tsd, edata, tctx, size);
malloc_mutex_lock(tsdn, tctx->tdata->lock);
szind_t szind = sz_size2index(usize);
malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
/*
* We need to do these map lookups while holding the lock, to avoid the
* possibility of races with prof_reset calls, which update the map and
* then acquire the lock. This actually still leaves a data race on the
* contents of the unbias map, but we have not yet gone through and
* atomic-ified the prof module, and compilers are not yet causing us
* issues. The key thing is to make sure that, if we read garbage data,
* the prof_reset call is about to mark our tctx as expired before any
* dumping of our corrupted output is attempted.
*/
size_t shifted_unbiased_cnt = prof_shifted_unbiased_cnt[szind];
size_t unbiased_bytes = prof_unbiased_sz[szind];
tctx->cnts.curobjs++;
tctx->cnts.curobjs_shifted_unbiased += shifted_unbiased_cnt;
tctx->cnts.curbytes += usize;
tctx->cnts.curbytes_unbiased += unbiased_bytes;
if (opt_prof_accum) {
tctx->cnts.accumobjs++;
tctx->cnts.accumobjs_shifted_unbiased += shifted_unbiased_cnt;
tctx->cnts.accumbytes += usize;
tctx->cnts.accumbytes_unbiased += unbiased_bytes;
}
bool record_recent = prof_recent_alloc_prepare(tsd, tctx);
tctx->prepared = false;
malloc_mutex_unlock(tsdn, tctx->tdata->lock);
}
static size_t
prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) {
assert(prof_logging_state == prof_logging_state_started);
malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx);
prof_bt_node_t dummy_node;
dummy_node.bt = *bt;
prof_bt_node_t *node;
/* See if this backtrace is already cached in the table. */
if (ckh_search(&log_bt_node_set, (void *)(&dummy_node),
(void **)(&node), NULL)) {
size_t sz = offsetof(prof_bt_node_t, vec) +
(bt->len * sizeof(void *));
prof_bt_node_t *new_node = (prof_bt_node_t *)
iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL,
true, arena_get(TSDN_NULL, 0, true), true);
if (log_bt_first == NULL) {
log_bt_first = new_node;
log_bt_last = new_node;
} else {
log_bt_last->next = new_node;
log_bt_last = new_node;
}
new_node->next = NULL;
new_node->index = log_bt_index;
/*
* Copy the backtrace: bt is inside a tdata or gctx, which
* might die before prof_log_stop is called.
*/
new_node->bt.len = bt->len;
memcpy(new_node->vec, bt->vec, bt->len * sizeof(void *));
new_node->bt.vec = new_node->vec;
log_bt_index++;
ckh_insert(tsd, &log_bt_node_set, (void *)new_node, NULL);
return new_node->index;
} else {
return node->index;
malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
if (record_recent) {
assert(tctx == edata_prof_tctx_get(edata));
prof_recent_alloc(tsd, edata, size, usize);
}
}
static size_t
prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) {
assert(prof_logging_state == prof_logging_state_started);
malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx);
prof_thr_node_t dummy_node;
dummy_node.thr_uid = thr_uid;
prof_thr_node_t *node;
/* See if this thread is already cached in the table. */
if (ckh_search(&log_thr_node_set, (void *)(&dummy_node),
(void **)(&node), NULL)) {
size_t sz = offsetof(prof_thr_node_t, name) + strlen(name) + 1;
prof_thr_node_t *new_node = (prof_thr_node_t *)
iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL,
true, arena_get(TSDN_NULL, 0, true), true);
if (log_thr_first == NULL) {
log_thr_first = new_node;
log_thr_last = new_node;
} else {
log_thr_last->next = new_node;
log_thr_last = new_node;
}
new_node->next = NULL;
new_node->index = log_thr_index;
new_node->thr_uid = thr_uid;
strcpy(new_node->name, name);
log_thr_index++;
ckh_insert(tsd, &log_thr_node_set, (void *)new_node, NULL);
return new_node->index;
} else {
return node->index;
if (opt_prof_stats) {
prof_stats_inc(tsd, szind, size);
}
}
static void
prof_try_log(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx) {
malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
prof_tdata_t *cons_tdata = prof_tdata_get(tsd, false);
if (cons_tdata == NULL) {
/*
* We decide not to log these allocations. cons_tdata will be
* NULL only when the current thread is in a weird state (e.g.
* it's being destroyed).
*/
return;
}
malloc_mutex_lock(tsd_tsdn(tsd), &log_mtx);
if (prof_logging_state != prof_logging_state_started) {
goto label_done;
}
if (!log_tables_initialized) {
bool err1 = ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS,
prof_bt_node_hash, prof_bt_node_keycomp);
bool err2 = ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS,
prof_thr_node_hash, prof_thr_node_keycomp);
if (err1 || err2) {
goto label_done;
}
log_tables_initialized = true;
}
nstime_t alloc_time = prof_alloc_time_get(tsd_tsdn(tsd), ptr,
(alloc_ctx_t *)NULL);
nstime_t free_time = NSTIME_ZERO_INITIALIZER;
nstime_update(&free_time);
size_t sz = sizeof(prof_alloc_node_t);
prof_alloc_node_t *new_node = (prof_alloc_node_t *)
iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true,
arena_get(TSDN_NULL, 0, true), true);
const char *prod_thr_name = (tctx->tdata->thread_name == NULL)?
"" : tctx->tdata->thread_name;
const char *cons_thr_name = prof_thread_name_get(tsd);
prof_bt_t bt;
/* Initialize the backtrace, using the buffer in tdata to store it. */
bt_init(&bt, cons_tdata->vec);
prof_backtrace(&bt);
prof_bt_t *cons_bt = &bt;
/* We haven't destroyed tctx yet, so gctx should be good to read. */
prof_bt_t *prod_bt = &tctx->gctx->bt;
new_node->next = NULL;
new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid,
prod_thr_name);
new_node->free_thr_ind = prof_log_thr_index(tsd, cons_tdata->thr_uid,
cons_thr_name);
new_node->alloc_bt_ind = prof_log_bt_index(tsd, prod_bt);
new_node->free_bt_ind = prof_log_bt_index(tsd, cons_bt);
new_node->alloc_time_ns = nstime_ns(&alloc_time);
new_node->free_time_ns = nstime_ns(&free_time);
new_node->usize = usize;
if (log_alloc_first == NULL) {
log_alloc_first = new_node;
log_alloc_last = new_node;
} else {
log_alloc_last->next = new_node;
log_alloc_last = new_node;
}
void
prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_info_t *prof_info) {
cassert(config_prof);
label_done:
malloc_mutex_unlock(tsd_tsdn(tsd), &log_mtx);
}
assert(prof_info != NULL);
prof_tctx_t *tctx = prof_info->alloc_tctx;
assert((uintptr_t)tctx > (uintptr_t)1U);
void
prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize,
prof_tctx_t *tctx) {
szind_t szind = sz_size2index(usize);
malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock);
assert(tctx->cnts.curobjs > 0);
assert(tctx->cnts.curbytes >= usize);
/*
* It's not correct to do equivalent asserts for unbiased bytes, because
* of the potential for races with prof.reset calls. The map contents
* should really be atomic, but we have not atomic-ified the prof module
* yet.
*/
tctx->cnts.curobjs--;
tctx->cnts.curobjs_shifted_unbiased -= prof_shifted_unbiased_cnt[szind];
tctx->cnts.curbytes -= usize;
tctx->cnts.curbytes_unbiased -= prof_unbiased_sz[szind];
prof_try_log(tsd, ptr, usize, tctx);
if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) {
prof_tctx_destroy(tsd, tctx);
} else {
malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock);
}
}
void
bt_init(prof_bt_t *bt, void **vec) {
cassert(config_prof);
bt->vec = vec;
bt->len = 0;
}
static void
prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {
cassert(config_prof);
assert(tdata == prof_tdata_get(tsd, false));
if (tdata != NULL) {
assert(!tdata->enq);
tdata->enq = true;
}
malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
}
static void
prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {
cassert(config_prof);
assert(tdata == prof_tdata_get(tsd, false));
malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
if (tdata != NULL) {
bool idump, gdump;
prof_try_log(tsd, usize, prof_info);
assert(tdata->enq);
tdata->enq = false;
idump = tdata->enq_idump;
tdata->enq_idump = false;
gdump = tdata->enq_gdump;
tdata->enq_gdump = false;
prof_tctx_try_destroy(tsd, tctx);
if (idump) {
prof_idump(tsd_tsdn(tsd));
}
if (gdump) {
prof_gdump(tsd_tsdn(tsd));
}
if (opt_prof_stats) {
prof_stats_dec(tsd, szind, prof_info->alloc_size);
}
}
#ifdef JEMALLOC_PROF_LIBUNWIND
void
prof_backtrace(prof_bt_t *bt) {
int nframes;
cassert(config_prof);
assert(bt->len == 0);
assert(bt->vec != NULL);
nframes = unw_backtrace(bt->vec, PROF_BT_MAX);
if (nframes <= 0) {
return;
prof_tctx_t *
prof_tctx_create(tsd_t *tsd) {
if (!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0) {
return NULL;
}
bt->len = nframes;
}
#elif (defined(JEMALLOC_PROF_LIBGCC))
static _Unwind_Reason_Code
prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) {
cassert(config_prof);
return _URC_NO_REASON;
}
static _Unwind_Reason_Code
prof_unwind_callback(struct _Unwind_Context *context, void *arg) {
prof_unwind_data_t *data = (prof_unwind_data_t *)arg;
void *ip;
cassert(config_prof);
ip = (void *)_Unwind_GetIP(context);
if (ip == NULL) {
return _URC_END_OF_STACK;
}
data->bt->vec[data->bt->len] = ip;
data->bt->len++;
if (data->bt->len == data->max) {
return _URC_END_OF_STACK;
prof_tdata_t *tdata = prof_tdata_get(tsd, true);
if (tdata == NULL) {
return NULL;
}
return _URC_NO_REASON;
prof_bt_t bt;
bt_init(&bt, tdata->vec);
prof_backtrace(tsd, &bt);
return prof_lookup(tsd, &bt);
}
void
prof_backtrace(prof_bt_t *bt) {
prof_unwind_data_t data = {bt, PROF_BT_MAX};
cassert(config_prof);
_Unwind_Backtrace(prof_unwind_callback, &data);
}
#elif (defined(JEMALLOC_PROF_GCC))
void
prof_backtrace(prof_bt_t *bt) {
#define BT_FRAME(i) \
if ((i) < PROF_BT_MAX) { \
void *p; \
if (__builtin_frame_address(i) == 0) { \
return; \
} \
p = __builtin_return_address(i); \
if (p == NULL) { \
return; \
} \
bt->vec[(i)] = p; \
bt->len = (i) + 1; \
} else { \
return; \
/*
* The bodies of this function and prof_leakcheck() are compiled out unless heap
* profiling is enabled, so that it is possible to compile jemalloc with
* floating point support completely disabled. Avoiding floating point code is
* important on memory-constrained systems, but it also enables a workaround for
* versions of glibc that don't properly save/restore floating point registers
* during dynamic lazy symbol loading (which internally calls into whatever
* malloc implementation happens to be integrated into the application). Note
* that some compilers (e.g. gcc 4.8) may use floating point registers for fast
* memory moves, so jemalloc must be compiled with such optimizations disabled
* (e.g.
* -mno-sse) in order for the workaround to be complete.
*/
uint64_t
prof_sample_new_event_wait(tsd_t *tsd) {
#ifdef JEMALLOC_PROF
if (lg_prof_sample == 0) {
return TE_MIN_START_WAIT;
}
cassert(config_prof);
BT_FRAME(0)
BT_FRAME(1)
BT_FRAME(2)
BT_FRAME(3)
BT_FRAME(4)
BT_FRAME(5)
BT_FRAME(6)
BT_FRAME(7)
BT_FRAME(8)
BT_FRAME(9)
BT_FRAME(10)
BT_FRAME(11)
BT_FRAME(12)
BT_FRAME(13)
BT_FRAME(14)
BT_FRAME(15)
BT_FRAME(16)
BT_FRAME(17)
BT_FRAME(18)
BT_FRAME(19)
BT_FRAME(20)
BT_FRAME(21)
BT_FRAME(22)
BT_FRAME(23)
BT_FRAME(24)
BT_FRAME(25)
BT_FRAME(26)
BT_FRAME(27)
BT_FRAME(28)
BT_FRAME(29)
BT_FRAME(30)
BT_FRAME(31)
BT_FRAME(32)
BT_FRAME(33)
BT_FRAME(34)
BT_FRAME(35)
BT_FRAME(36)
BT_FRAME(37)
BT_FRAME(38)
BT_FRAME(39)
BT_FRAME(40)
BT_FRAME(41)
BT_FRAME(42)
BT_FRAME(43)
BT_FRAME(44)
BT_FRAME(45)
BT_FRAME(46)
BT_FRAME(47)
BT_FRAME(48)
BT_FRAME(49)
BT_FRAME(50)
BT_FRAME(51)
BT_FRAME(52)
BT_FRAME(53)
BT_FRAME(54)
BT_FRAME(55)
BT_FRAME(56)
BT_FRAME(57)
BT_FRAME(58)
BT_FRAME(59)
BT_FRAME(60)
BT_FRAME(61)
BT_FRAME(62)
BT_FRAME(63)
BT_FRAME(64)
BT_FRAME(65)
BT_FRAME(66)
BT_FRAME(67)
BT_FRAME(68)
BT_FRAME(69)
BT_FRAME(70)
BT_FRAME(71)
BT_FRAME(72)
BT_FRAME(73)
BT_FRAME(74)
BT_FRAME(75)
BT_FRAME(76)
BT_FRAME(77)
BT_FRAME(78)
BT_FRAME(79)
BT_FRAME(80)
BT_FRAME(81)
BT_FRAME(82)
BT_FRAME(83)
BT_FRAME(84)
BT_FRAME(85)
BT_FRAME(86)
BT_FRAME(87)
BT_FRAME(88)
BT_FRAME(89)
BT_FRAME(90)
BT_FRAME(91)
BT_FRAME(92)
BT_FRAME(93)
BT_FRAME(94)
BT_FRAME(95)
BT_FRAME(96)
BT_FRAME(97)
BT_FRAME(98)
BT_FRAME(99)
BT_FRAME(100)
BT_FRAME(101)
BT_FRAME(102)
BT_FRAME(103)
BT_FRAME(104)
BT_FRAME(105)
BT_FRAME(106)
BT_FRAME(107)
BT_FRAME(108)
BT_FRAME(109)
BT_FRAME(110)
BT_FRAME(111)
BT_FRAME(112)
BT_FRAME(113)
BT_FRAME(114)
BT_FRAME(115)
BT_FRAME(116)
BT_FRAME(117)
BT_FRAME(118)
BT_FRAME(119)
BT_FRAME(120)
BT_FRAME(121)
BT_FRAME(122)
BT_FRAME(123)
BT_FRAME(124)
BT_FRAME(125)
BT_FRAME(126)
BT_FRAME(127)
#undef BT_FRAME
}
/*
* Compute sample interval as a geometrically distributed random
* variable with mean (2^lg_prof_sample).
*
* __ __
* | log(u) | 1
* bytes_until_sample = | -------- |, where p = ---------------
* | log(1-p) | lg_prof_sample
* 2
*
* For more information on the math, see:
*
* Non-Uniform Random Variate Generation
* Luc Devroye
* Springer-Verlag, New York, 1986
* pp 500
* (http://luc.devroye.org/rnbookindex.html)
*
* In the actual computation, there's a non-zero probability that our
* pseudo random number generator generates an exact 0, and to avoid
* log(0), we set u to 1.0 in case r is 0. Therefore u effectively is
* uniformly distributed in (0, 1] instead of [0, 1). Further, rather
* than taking the ceiling, we take the floor and then add 1, since
* otherwise bytes_until_sample would be 0 if u is exactly 1.0.
*/
uint64_t r = prng_lg_range_u64(tsd_prng_statep_get(tsd), 53);
double u = (r == 0U) ? 1.0 : (double)r * (1.0/9007199254740992.0L);
return (uint64_t)(log(u) /
log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
+ (uint64_t)1U;
#else
void
prof_backtrace(prof_bt_t *bt) {
cassert(config_prof);
not_reached();
}
return TE_MAX_START_WAIT;
#endif
static malloc_mutex_t *
prof_gctx_mutex_choose(void) {
unsigned ngctxs = atomic_fetch_add_u(&cum_gctxs, 1, ATOMIC_RELAXED);
return &gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS];
}
static malloc_mutex_t *
prof_tdata_mutex_choose(uint64_t thr_uid) {
return &tdata_locks[thr_uid % PROF_NTDATA_LOCKS];
}
static prof_gctx_t *
prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) {
/*
* Create a single allocation that has space for vec of length bt->len.
*/
size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *));
prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size,
sz_size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true),
true);
if (gctx == NULL) {
return NULL;
}
gctx->lock = prof_gctx_mutex_choose();
/*
* Set nlimbo to 1, in order to avoid a race condition with
* prof_tctx_destroy()/prof_gctx_try_destroy().
*/
gctx->nlimbo = 1;
tctx_tree_new(&gctx->tctxs);
/* Duplicate bt. */
memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
gctx->bt.vec = gctx->vec;
gctx->bt.len = bt->len;
return gctx;
}
static void
prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
prof_tdata_t *tdata) {
cassert(config_prof);
uint64_t
prof_sample_postponed_event_wait(tsd_t *tsd) {
/*
* Check that gctx is still unused by any thread cache before destroying
* it. prof_lookup() increments gctx->nlimbo in order to avoid a race
* condition with this function, as does prof_tctx_destroy() in order to
* avoid a race between the main body of prof_tctx_destroy() and entry
* into this function.
* The postponed wait time for prof sample event is computed as if we
* want a new wait time (i.e. as if the event were triggered). If we
* instead postpone to the immediate next allocation, like how we're
* handling the other events, then we can have sampling bias, if e.g.
* the allocation immediately following a reentrancy always comes from
* the same stack trace.
*/
prof_enter(tsd, tdata_self);
malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
assert(gctx->nlimbo != 0);
if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
/* Remove gctx from bt2gctx. */
if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) {
not_reached();
}
prof_leave(tsd, tdata_self);
/* Destroy gctx. */
malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
idalloctm(tsd_tsdn(tsd), gctx, NULL, NULL, true, true);
} else {
/*
* Compensate for increment in prof_tctx_destroy() or
* prof_lookup().
*/
gctx->nlimbo--;
malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
prof_leave(tsd, tdata_self);
}
}
static bool
prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx) {
malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
if (opt_prof_accum) {
return false;
}
if (tctx->cnts.curobjs != 0) {
return false;
}
if (tctx->prepared) {
return false;
}
return true;
}
static bool
prof_gctx_should_destroy(prof_gctx_t *gctx) {
if (opt_prof_accum) {
return false;
}
if (!tctx_tree_empty(&gctx->tctxs)) {
return false;
}
if (gctx->nlimbo != 0) {
return false;
}
return true;
}
static void
prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
prof_tdata_t *tdata = tctx->tdata;
prof_gctx_t *gctx = tctx->gctx;
bool destroy_tdata, destroy_tctx, destroy_gctx;
malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
assert(tctx->cnts.curobjs == 0);
assert(tctx->cnts.curbytes == 0);
assert(!opt_prof_accum);
assert(tctx->cnts.accumobjs == 0);
assert(tctx->cnts.accumbytes == 0);
ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false);
malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
switch (tctx->state) {
case prof_tctx_state_nominal:
tctx_tree_remove(&gctx->tctxs, tctx);
destroy_tctx = true;
if (prof_gctx_should_destroy(gctx)) {
/*
* Increment gctx->nlimbo in order to keep another
* thread from winning the race to destroy gctx while
* this one has gctx->lock dropped. Without this, it
* would be possible for another thread to:
*
* 1) Sample an allocation associated with gctx.
* 2) Deallocate the sampled object.
* 3) Successfully prof_gctx_try_destroy(gctx).
*
* The result would be that gctx no longer exists by the
* time this thread accesses it in
* prof_gctx_try_destroy().
*/
gctx->nlimbo++;
destroy_gctx = true;
} else {
destroy_gctx = false;
}
break;
case prof_tctx_state_dumping:
/*
* A dumping thread needs tctx to remain valid until dumping
* has finished. Change state such that the dumping thread will
* complete destruction during a late dump iteration phase.
*/
tctx->state = prof_tctx_state_purgatory;
destroy_tctx = false;
destroy_gctx = false;
break;
default:
not_reached();
destroy_tctx = false;
destroy_gctx = false;
}
malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
if (destroy_gctx) {
prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx,
tdata);
}
malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock);
if (destroy_tdata) {
prof_tdata_destroy(tsd, tdata, false);
}
if (destroy_tctx) {
idalloctm(tsd_tsdn(tsd), tctx, NULL, NULL, true, true);
}
}
static bool
prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) {
union {
prof_gctx_t *p;
void *v;
} gctx, tgctx;
union {
prof_bt_t *p;
void *v;
} btkey;
bool new_gctx;
prof_enter(tsd, tdata);
if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
/* bt has never been seen before. Insert it. */
prof_leave(tsd, tdata);
tgctx.p = prof_gctx_create(tsd_tsdn(tsd), bt);
if (tgctx.v == NULL) {
return true;
}
prof_enter(tsd, tdata);
if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
gctx.p = tgctx.p;
btkey.p = &gctx.p->bt;
if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
/* OOM. */
prof_leave(tsd, tdata);
idalloctm(tsd_tsdn(tsd), gctx.v, NULL, NULL,
true, true);
return true;
}
new_gctx = true;
} else {
new_gctx = false;
}
} else {
tgctx.v = NULL;
new_gctx = false;
}
if (!new_gctx) {
/*
* Increment nlimbo, in order to avoid a race condition with
* prof_tctx_destroy()/prof_gctx_try_destroy().
*/
malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock);
gctx.p->nlimbo++;
malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock);
new_gctx = false;
if (tgctx.v != NULL) {
/* Lost race to insert. */
idalloctm(tsd_tsdn(tsd), tgctx.v, NULL, NULL, true,
true);
}
}
prof_leave(tsd, tdata);
*p_btkey = btkey.v;
*p_gctx = gctx.p;
*p_new_gctx = new_gctx;
return false;
}
prof_tctx_t *
prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
union {
prof_tctx_t *p;
void *v;
} ret;
prof_tdata_t *tdata;
bool not_found;
cassert(config_prof);
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL) {
return NULL;
}
malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
if (!not_found) { /* Note double negative! */
ret.p->prepared = true;
}
malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
if (not_found) {
void *btkey;
prof_gctx_t *gctx;
bool new_gctx, error;
/*
* This thread's cache lacks bt. Look for it in the global
* cache.
*/
if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
&new_gctx)) {
return NULL;
}
/* Link a prof_tctx_t into gctx for this thread. */
ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t),
sz_size2index(sizeof(prof_tctx_t)), false, NULL, true,
arena_ichoose(tsd, NULL), true);
if (ret.p == NULL) {
if (new_gctx) {
prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
}
return NULL;
}
ret.p->tdata = tdata;
ret.p->thr_uid = tdata->thr_uid;
ret.p->thr_discrim = tdata->thr_discrim;
memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
ret.p->gctx = gctx;
ret.p->tctx_uid = tdata->tctx_uid_next++;
ret.p->prepared = true;
ret.p->state = prof_tctx_state_initializing;
malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
if (error) {
if (new_gctx) {
prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
}
idalloctm(tsd_tsdn(tsd), ret.v, NULL, NULL, true, true);
return NULL;
}
malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
ret.p->state = prof_tctx_state_nominal;
tctx_tree_insert(&gctx->tctxs, ret.p);
gctx->nlimbo--;
malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
}
return ret.p;
}
/*
* The bodies of this function and prof_leakcheck() are compiled out unless heap
* profiling is enabled, so that it is possible to compile jemalloc with
* floating point support completely disabled. Avoiding floating point code is
* important on memory-constrained systems, but it also enables a workaround for
* versions of glibc that don't properly save/restore floating point registers
* during dynamic lazy symbol loading (which internally calls into whatever
* malloc implementation happens to be integrated into the application). Note
* that some compilers (e.g. gcc 4.8) may use floating point registers for fast
* memory moves, so jemalloc must be compiled with such optimizations disabled
* (e.g.
* -mno-sse) in order for the workaround to be complete.
*/
void
prof_sample_threshold_update(prof_tdata_t *tdata) {
#ifdef JEMALLOC_PROF
if (!config_prof) {
return;
}
if (lg_prof_sample == 0) {
tsd_bytes_until_sample_set(tsd_fetch(), 0);
return;
}
/*
* Compute sample interval as a geometrically distributed random
* variable with mean (2^lg_prof_sample).
*
* __ __
* | log(u) | 1
* tdata->bytes_until_sample = | -------- |, where p = ---------------
* | log(1-p) | lg_prof_sample
* 2
*
* For more information on the math, see:
*
* Non-Uniform Random Variate Generation
* Luc Devroye
* Springer-Verlag, New York, 1986
* pp 500
* (http://luc.devroye.org/rnbookindex.html)
*/
uint64_t r = prng_lg_range_u64(&tdata->prng_state, 53);
double u = (double)r * (1.0/9007199254740992.0L);
uint64_t bytes_until_sample = (uint64_t)(log(u) /
log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
+ (uint64_t)1U;
if (bytes_until_sample > SSIZE_MAX) {
bytes_until_sample = SSIZE_MAX;
}
tsd_bytes_until_sample_set(tsd_fetch(), bytes_until_sample);
#endif
}
#ifdef JEMALLOC_JET
static prof_tdata_t *
prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
void *arg) {
size_t *tdata_count = (size_t *)arg;
(*tdata_count)++;
return NULL;
}
size_t
prof_tdata_count(void) {
size_t tdata_count = 0;
tsdn_t *tsdn;
tsdn = tsdn_fetch();
malloc_mutex_lock(tsdn, &tdatas_mtx);
tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
(void *)&tdata_count);
malloc_mutex_unlock(tsdn, &tdatas_mtx);
return tdata_count;
}
size_t
prof_bt_count(void) {
size_t bt_count;
tsd_t *tsd;
prof_tdata_t *tdata;
tsd = tsd_fetch();
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL) {
return 0;
}
malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
bt_count = ckh_count(&bt2gctx);
malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
return bt_count;
}
#endif
static int
prof_dump_open_impl(bool propagate_err, const char *filename) {
int fd;
fd = creat(filename, 0644);
if (fd == -1 && !propagate_err) {
malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n",
filename);
if (opt_abort) {
abort();
}
}
return fd;
}
prof_dump_open_t *JET_MUTABLE prof_dump_open = prof_dump_open_impl;
static bool
prof_dump_flush(bool propagate_err) {
bool ret = false;
ssize_t err;
cassert(config_prof);
err = malloc_write_fd(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
if (err == -1) {
if (!propagate_err) {
malloc_write("<jemalloc>: write() failed during heap "
"profile flush\n");
if (opt_abort) {
abort();
}
}
ret = true;
}
prof_dump_buf_end = 0;
return ret;
}
static bool
prof_dump_close(bool propagate_err) {
bool ret;
assert(prof_dump_fd != -1);
ret = prof_dump_flush(propagate_err);
close(prof_dump_fd);
prof_dump_fd = -1;
return ret;
}
static bool
prof_dump_write(bool propagate_err, const char *s) {
size_t i, slen, n;
cassert(config_prof);
i = 0;
slen = strlen(s);
while (i < slen) {
/* Flush the buffer if it is full. */
if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
if (prof_dump_flush(propagate_err) && propagate_err) {
return true;
}
}
if (prof_dump_buf_end + slen - i <= PROF_DUMP_BUFSIZE) {
/* Finish writing. */
n = slen - i;
} else {
/* Write as much of s as will fit. */
n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
}
memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
prof_dump_buf_end += n;
i += n;
}
assert(i == slen);
return false;
}
JEMALLOC_FORMAT_PRINTF(2, 3)
static bool
prof_dump_printf(bool propagate_err, const char *format, ...) {
bool ret;
va_list ap;
char buf[PROF_PRINTF_BUFSIZE];
va_start(ap, format);
malloc_vsnprintf(buf, sizeof(buf), format, ap);
va_end(ap);
ret = prof_dump_write(propagate_err, buf);
return ret;
}
static void
prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) {
malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
malloc_mutex_lock(tsdn, tctx->gctx->lock);
switch (tctx->state) {
case prof_tctx_state_initializing:
malloc_mutex_unlock(tsdn, tctx->gctx->lock);
return;
case prof_tctx_state_nominal:
tctx->state = prof_tctx_state_dumping;
malloc_mutex_unlock(tsdn, tctx->gctx->lock);
memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
if (opt_prof_accum) {
tdata->cnt_summed.accumobjs +=
tctx->dump_cnts.accumobjs;
tdata->cnt_summed.accumbytes +=
tctx->dump_cnts.accumbytes;
}
break;
case prof_tctx_state_dumping:
case prof_tctx_state_purgatory:
not_reached();
}
}
static void
prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) {
malloc_mutex_assert_owner(tsdn, gctx->lock);
gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
if (opt_prof_accum) {
gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
}
}
static prof_tctx_t *
prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
tsdn_t *tsdn = (tsdn_t *)arg;
malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
switch (tctx->state) {
case prof_tctx_state_nominal:
/* New since dumping started; ignore. */
break;
case prof_tctx_state_dumping:
case prof_tctx_state_purgatory:
prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx);
break;
default:
not_reached();
}
return NULL;
}
struct prof_tctx_dump_iter_arg_s {
tsdn_t *tsdn;
bool propagate_err;
};
static prof_tctx_t *
prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) {
struct prof_tctx_dump_iter_arg_s *arg =
(struct prof_tctx_dump_iter_arg_s *)opaque;
malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock);
switch (tctx->state) {
case prof_tctx_state_initializing:
case prof_tctx_state_nominal:
/* Not captured by this dump. */
break;
case prof_tctx_state_dumping:
case prof_tctx_state_purgatory:
if (prof_dump_printf(arg->propagate_err,
" t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": "
"%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs,
tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs,
tctx->dump_cnts.accumbytes)) {
return tctx;
}
break;
default:
not_reached();
}
return NULL;
}
static prof_tctx_t *
prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
tsdn_t *tsdn = (tsdn_t *)arg;
prof_tctx_t *ret;
malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
switch (tctx->state) {
case prof_tctx_state_nominal:
/* New since dumping started; ignore. */
break;
case prof_tctx_state_dumping:
tctx->state = prof_tctx_state_nominal;
break;
case prof_tctx_state_purgatory:
ret = tctx;
goto label_return;
default:
not_reached();
}
ret = NULL;
label_return:
return ret;
}
static void
prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) {
cassert(config_prof);
malloc_mutex_lock(tsdn, gctx->lock);
/*
* Increment nlimbo so that gctx won't go away before dump.
* Additionally, link gctx into the dump list so that it is included in
* prof_dump()'s second pass.
*/
gctx->nlimbo++;
gctx_tree_insert(gctxs, gctx);
memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
malloc_mutex_unlock(tsdn, gctx->lock);
}
struct prof_gctx_merge_iter_arg_s {
tsdn_t *tsdn;
size_t leak_ngctx;
};
static prof_gctx_t *
prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
struct prof_gctx_merge_iter_arg_s *arg =
(struct prof_gctx_merge_iter_arg_s *)opaque;
malloc_mutex_lock(arg->tsdn, gctx->lock);
tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter,
(void *)arg->tsdn);
if (gctx->cnt_summed.curobjs != 0) {
arg->leak_ngctx++;
}
malloc_mutex_unlock(arg->tsdn, gctx->lock);
return NULL;
}
static void
prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) {
prof_tdata_t *tdata = prof_tdata_get(tsd, false);
prof_gctx_t *gctx;
/*
* Standard tree iteration won't work here, because as soon as we
* decrement gctx->nlimbo and unlock gctx, another thread can
* concurrently destroy it, which will corrupt the tree. Therefore,
* tear down the tree one node at a time during iteration.
*/
while ((gctx = gctx_tree_first(gctxs)) != NULL) {
gctx_tree_remove(gctxs, gctx);
malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
{
prof_tctx_t *next;
next = NULL;
do {
prof_tctx_t *to_destroy =
tctx_tree_iter(&gctx->tctxs, next,
prof_tctx_finish_iter,
(void *)tsd_tsdn(tsd));
if (to_destroy != NULL) {
next = tctx_tree_next(&gctx->tctxs,
to_destroy);
tctx_tree_remove(&gctx->tctxs,
to_destroy);
idalloctm(tsd_tsdn(tsd), to_destroy,
NULL, NULL, true, true);
} else {
next = NULL;
}
} while (next != NULL);
}
gctx->nlimbo--;
if (prof_gctx_should_destroy(gctx)) {
gctx->nlimbo++;
malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
} else {
malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
}
}
}
struct prof_tdata_merge_iter_arg_s {
tsdn_t *tsdn;
prof_cnt_t cnt_all;
};
static prof_tdata_t *
prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
void *opaque) {
struct prof_tdata_merge_iter_arg_s *arg =
(struct prof_tdata_merge_iter_arg_s *)opaque;
malloc_mutex_lock(arg->tsdn, tdata->lock);
if (!tdata->expired) {
size_t tabind;
union {
prof_tctx_t *p;
void *v;
} tctx;
tdata->dumping = true;
memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
&tctx.v);) {
prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);
}
arg->cnt_all.curobjs += tdata->cnt_summed.curobjs;
arg->cnt_all.curbytes += tdata->cnt_summed.curbytes;
if (opt_prof_accum) {
arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs;
arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes;
}
} else {
tdata->dumping = false;
}
malloc_mutex_unlock(arg->tsdn, tdata->lock);
return NULL;
}
static prof_tdata_t *
prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
void *arg) {
bool propagate_err = *(bool *)arg;
if (!tdata->dumping) {
return NULL;
}
if (prof_dump_printf(propagate_err,
" t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n",
tdata->thr_uid, tdata->cnt_summed.curobjs,
tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,
tdata->cnt_summed.accumbytes,
(tdata->thread_name != NULL) ? " " : "",
(tdata->thread_name != NULL) ? tdata->thread_name : "")) {
return tdata;
}
return NULL;
}
static bool
prof_dump_header_impl(tsdn_t *tsdn, bool propagate_err,
const prof_cnt_t *cnt_all) {
bool ret;
if (prof_dump_printf(propagate_err,
"heap_v2/%"FMTu64"\n"
" t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
((uint64_t)1U << lg_prof_sample), cnt_all->curobjs,
cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) {
return true;
}
malloc_mutex_lock(tsdn, &tdatas_mtx);
ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter,
(void *)&propagate_err) != NULL);
malloc_mutex_unlock(tsdn, &tdatas_mtx);
return ret;
}
prof_dump_header_t *JET_MUTABLE prof_dump_header = prof_dump_header_impl;
static bool
prof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx,
const prof_bt_t *bt, prof_gctx_tree_t *gctxs) {
bool ret;
unsigned i;
struct prof_tctx_dump_iter_arg_s prof_tctx_dump_iter_arg;
cassert(config_prof);
malloc_mutex_assert_owner(tsdn, gctx->lock);
/* Avoid dumping such gctx's that have no useful data. */
if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
(opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
assert(gctx->cnt_summed.curobjs == 0);
assert(gctx->cnt_summed.curbytes == 0);
assert(gctx->cnt_summed.accumobjs == 0);
assert(gctx->cnt_summed.accumbytes == 0);
ret = false;
goto label_return;
}
if (prof_dump_printf(propagate_err, "@")) {
ret = true;
goto label_return;
}
for (i = 0; i < bt->len; i++) {
if (prof_dump_printf(propagate_err, " %#"FMTxPTR,
(uintptr_t)bt->vec[i])) {
ret = true;
goto label_return;
}
}
if (prof_dump_printf(propagate_err,
"\n"
" t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes,
gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) {
ret = true;
goto label_return;
}
prof_tctx_dump_iter_arg.tsdn = tsdn;
prof_tctx_dump_iter_arg.propagate_err = propagate_err;
if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter,
(void *)&prof_tctx_dump_iter_arg) != NULL) {
ret = true;
goto label_return;
}
ret = false;
label_return:
return ret;
}
#ifndef _WIN32
JEMALLOC_FORMAT_PRINTF(1, 2)
static int
prof_open_maps(const char *format, ...) {
int mfd;
va_list ap;
char filename[PATH_MAX + 1];
va_start(ap, format);
malloc_vsnprintf(filename, sizeof(filename), format, ap);
va_end(ap);
#if defined(O_CLOEXEC)
mfd = open(filename, O_RDONLY | O_CLOEXEC);
#else
mfd = open(filename, O_RDONLY);
if (mfd != -1) {
fcntl(mfd, F_SETFD, fcntl(mfd, F_GETFD) | FD_CLOEXEC);
}
#endif
return mfd;
}
#endif
static int
prof_getpid(void) {
#ifdef _WIN32
return GetCurrentProcessId();
#else
return getpid();
#endif
}
static bool
prof_dump_maps(bool propagate_err) {
bool ret;
int mfd;
cassert(config_prof);
#ifdef __FreeBSD__
mfd = prof_open_maps("/proc/curproc/map");
#elif defined(_WIN32)
mfd = -1; // Not implemented
#else
{
int pid = prof_getpid();
mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid);
if (mfd == -1) {
mfd = prof_open_maps("/proc/%d/maps", pid);
}
}
#endif
if (mfd != -1) {
ssize_t nread;
if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") &&
propagate_err) {
ret = true;
goto label_return;
}
nread = 0;
do {
prof_dump_buf_end += nread;
if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
/* Make space in prof_dump_buf before read(). */
if (prof_dump_flush(propagate_err) &&
propagate_err) {
ret = true;
goto label_return;
}
}
nread = malloc_read_fd(mfd,
&prof_dump_buf[prof_dump_buf_end], PROF_DUMP_BUFSIZE
- prof_dump_buf_end);
} while (nread > 0);
} else {
ret = true;
goto label_return;
}
ret = false;
label_return:
if (mfd != -1) {
close(mfd);
}
return ret;
}
/*
* See prof_sample_threshold_update() comment for why the body of this function
* is conditionally compiled.
*/
static void
prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx,
const char *filename) {
#ifdef JEMALLOC_PROF
/*
* Scaling is equivalent AdjustSamples() in jeprof, but the result may
* differ slightly from what jeprof reports, because here we scale the
* summary values, whereas jeprof scales each context individually and
* reports the sums of the scaled values.
*/
if (cnt_all->curbytes != 0) {
double sample_period = (double)((uint64_t)1 << lg_prof_sample);
double ratio = (((double)cnt_all->curbytes) /
(double)cnt_all->curobjs) / sample_period;
double scale_factor = 1.0 / (1.0 - exp(-ratio));
uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes)
* scale_factor);
uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) *
scale_factor);
malloc_printf("<jemalloc>: Leak approximation summary: ~%"FMTu64
" byte%s, ~%"FMTu64" object%s, >= %zu context%s\n",
curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs !=
1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : "");
malloc_printf(
"<jemalloc>: Run jeprof on \"%s\" for leak detail\n",
filename);
}
#endif
}
struct prof_gctx_dump_iter_arg_s {
tsdn_t *tsdn;
bool propagate_err;
};
static prof_gctx_t *
prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
prof_gctx_t *ret;
struct prof_gctx_dump_iter_arg_s *arg =
(struct prof_gctx_dump_iter_arg_s *)opaque;
malloc_mutex_lock(arg->tsdn, gctx->lock);
if (prof_dump_gctx(arg->tsdn, arg->propagate_err, gctx, &gctx->bt,
gctxs)) {
ret = gctx;
goto label_return;
}
ret = NULL;
label_return:
malloc_mutex_unlock(arg->tsdn, gctx->lock);
return ret;
}
static void
prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata,
struct prof_tdata_merge_iter_arg_s *prof_tdata_merge_iter_arg,
struct prof_gctx_merge_iter_arg_s *prof_gctx_merge_iter_arg,
prof_gctx_tree_t *gctxs) {
size_t tabind;
union {
prof_gctx_t *p;
void *v;
} gctx;
prof_enter(tsd, tdata);
/*
* Put gctx's in limbo and clear their counters in preparation for
* summing.
*/
gctx_tree_new(gctxs);
for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) {
prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, gctxs);
}
/*
* Iterate over tdatas, and for the non-expired ones snapshot their tctx
* stats and merge them into the associated gctx's.
*/
prof_tdata_merge_iter_arg->tsdn = tsd_tsdn(tsd);
memset(&prof_tdata_merge_iter_arg->cnt_all, 0, sizeof(prof_cnt_t));
malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter,
(void *)prof_tdata_merge_iter_arg);
malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
/* Merge tctx stats into gctx's. */
prof_gctx_merge_iter_arg->tsdn = tsd_tsdn(tsd);
prof_gctx_merge_iter_arg->leak_ngctx = 0;
gctx_tree_iter(gctxs, NULL, prof_gctx_merge_iter,
(void *)prof_gctx_merge_iter_arg);
prof_leave(tsd, tdata);
}
static bool
prof_dump_file(tsd_t *tsd, bool propagate_err, const char *filename,
bool leakcheck, prof_tdata_t *tdata,
struct prof_tdata_merge_iter_arg_s *prof_tdata_merge_iter_arg,
struct prof_gctx_merge_iter_arg_s *prof_gctx_merge_iter_arg,
struct prof_gctx_dump_iter_arg_s *prof_gctx_dump_iter_arg,
prof_gctx_tree_t *gctxs) {
/* Create dump file. */
if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) {
return true;
}
/* Dump profile header. */
if (prof_dump_header(tsd_tsdn(tsd), propagate_err,
&prof_tdata_merge_iter_arg->cnt_all)) {
goto label_write_error;
}
/* Dump per gctx profile stats. */
prof_gctx_dump_iter_arg->tsdn = tsd_tsdn(tsd);
prof_gctx_dump_iter_arg->propagate_err = propagate_err;
if (gctx_tree_iter(gctxs, NULL, prof_gctx_dump_iter,
(void *)prof_gctx_dump_iter_arg) != NULL) {
goto label_write_error;
}
/* Dump /proc/<pid>/maps if possible. */
if (prof_dump_maps(propagate_err)) {
goto label_write_error;
}
if (prof_dump_close(propagate_err)) {
return true;
}
return false;
label_write_error:
prof_dump_close(propagate_err);
return true;
}
static bool
prof_dump(tsd_t *tsd, bool propagate_err, const char *filename,
bool leakcheck) {
cassert(config_prof);
assert(tsd_reentrancy_level_get(tsd) == 0);
prof_tdata_t * tdata = prof_tdata_get(tsd, true);
if (tdata == NULL) {
return true;
}
pre_reentrancy(tsd, NULL);
malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
prof_gctx_tree_t gctxs;
struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg;
struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg;
struct prof_gctx_dump_iter_arg_s prof_gctx_dump_iter_arg;
prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,
&prof_gctx_merge_iter_arg, &gctxs);
bool err = prof_dump_file(tsd, propagate_err, filename, leakcheck, tdata,
&prof_tdata_merge_iter_arg, &prof_gctx_merge_iter_arg,
&prof_gctx_dump_iter_arg, &gctxs);
prof_gctx_finish(tsd, &gctxs);
malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
post_reentrancy(tsd);
if (err) {
return true;
}
if (leakcheck) {
prof_leakcheck(&prof_tdata_merge_iter_arg.cnt_all,
prof_gctx_merge_iter_arg.leak_ngctx, filename);
}
return false;
}
#ifdef JEMALLOC_JET
void
prof_cnt_all(uint64_t *curobjs, uint64_t *curbytes, uint64_t *accumobjs,
uint64_t *accumbytes) {
tsd_t *tsd;
prof_tdata_t *tdata;
struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg;
struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg;
prof_gctx_tree_t gctxs;
tsd = tsd_fetch();
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL) {
if (curobjs != NULL) {
*curobjs = 0;
}
if (curbytes != NULL) {
*curbytes = 0;
}
if (accumobjs != NULL) {
*accumobjs = 0;
}
if (accumbytes != NULL) {
*accumbytes = 0;
}
return;
}
prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,
&prof_gctx_merge_iter_arg, &gctxs);
prof_gctx_finish(tsd, &gctxs);
if (curobjs != NULL) {
*curobjs = prof_tdata_merge_iter_arg.cnt_all.curobjs;
}
if (curbytes != NULL) {
*curbytes = prof_tdata_merge_iter_arg.cnt_all.curbytes;
}
if (accumobjs != NULL) {
*accumobjs = prof_tdata_merge_iter_arg.cnt_all.accumobjs;
}
if (accumbytes != NULL) {
*accumbytes = prof_tdata_merge_iter_arg.cnt_all.accumbytes;
}
}
#endif
#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1)
#define VSEQ_INVALID UINT64_C(0xffffffffffffffff)
static void
prof_dump_filename(char *filename, char v, uint64_t vseq) {
cassert(config_prof);
if (vseq != VSEQ_INVALID) {
/* "<prefix>.<pid>.<seq>.v<vseq>.heap" */
malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
"%s.%d.%"FMTu64".%c%"FMTu64".heap",
opt_prof_prefix, prof_getpid(), prof_dump_seq, v, vseq);
} else {
/* "<prefix>.<pid>.<seq>.<v>.heap" */
malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
"%s.%d.%"FMTu64".%c.heap",
opt_prof_prefix, prof_getpid(), prof_dump_seq, v);
}
prof_dump_seq++;
}
static void
prof_fdump(void) {
tsd_t *tsd;
char filename[DUMP_FILENAME_BUFSIZE];
cassert(config_prof);
assert(opt_prof_final);
assert(opt_prof_prefix[0] != '\0');
if (!prof_booted) {
return;
}
tsd = tsd_fetch();
assert(tsd_reentrancy_level_get(tsd) == 0);
malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
prof_dump_filename(filename, 'f', VSEQ_INVALID);
malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
prof_dump(tsd, false, filename, opt_prof_leak);
}
bool
prof_accum_init(tsdn_t *tsdn, prof_accum_t *prof_accum) {
cassert(config_prof);
#ifndef JEMALLOC_ATOMIC_U64
if (malloc_mutex_init(&prof_accum->mtx, "prof_accum",
WITNESS_RANK_PROF_ACCUM, malloc_mutex_rank_exclusive)) {
return true;
}
prof_accum->accumbytes = 0;
#else
atomic_store_u64(&prof_accum->accumbytes, 0, ATOMIC_RELAXED);
#endif
return false;
}
void
prof_idump(tsdn_t *tsdn) {
tsd_t *tsd;
prof_tdata_t *tdata;
cassert(config_prof);
if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) {
return;
}
tsd = tsdn_tsd(tsdn);
if (tsd_reentrancy_level_get(tsd) > 0) {
return;
}
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL) {
return;
}
if (tdata->enq) {
tdata->enq_idump = true;
return;
}
if (opt_prof_prefix[0] != '\0') {
char filename[PATH_MAX + 1];
malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
prof_dump_filename(filename, 'i', prof_dump_iseq);
prof_dump_iseq++;
malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
prof_dump(tsd, false, filename, false);
}
}
bool
prof_mdump(tsd_t *tsd, const char *filename) {
cassert(config_prof);
assert(tsd_reentrancy_level_get(tsd) == 0);
if (!opt_prof || !prof_booted) {
return true;
}
char filename_buf[DUMP_FILENAME_BUFSIZE];
if (filename == NULL) {
/* No filename specified, so automatically generate one. */
if (opt_prof_prefix[0] == '\0') {
return true;
}
malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
prof_dump_filename(filename_buf, 'm', prof_dump_mseq);
prof_dump_mseq++;
malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx);
filename = filename_buf;
}
return prof_dump(tsd, true, filename, false);
}
void
prof_gdump(tsdn_t *tsdn) {
tsd_t *tsd;
prof_tdata_t *tdata;
cassert(config_prof);
if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) {
return;
}
tsd = tsdn_tsd(tsdn);
if (tsd_reentrancy_level_get(tsd) > 0) {
return;
}
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL) {
return;
}
if (tdata->enq) {
tdata->enq_gdump = true;
return;
}
if (opt_prof_prefix[0] != '\0') {
char filename[DUMP_FILENAME_BUFSIZE];
malloc_mutex_lock(tsdn, &prof_dump_seq_mtx);
prof_dump_filename(filename, 'u', prof_dump_useq);
prof_dump_useq++;
malloc_mutex_unlock(tsdn, &prof_dump_seq_mtx);
prof_dump(tsd, false, filename, false);
}
}
static void
prof_bt_hash(const void *key, size_t r_hash[2]) {
prof_bt_t *bt = (prof_bt_t *)key;
cassert(config_prof);
hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
}
static bool
prof_bt_keycomp(const void *k1, const void *k2) {
const prof_bt_t *bt1 = (prof_bt_t *)k1;
const prof_bt_t *bt2 = (prof_bt_t *)k2;
cassert(config_prof);
if (bt1->len != bt2->len) {
return false;
}
return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
}
static void
prof_bt_node_hash(const void *key, size_t r_hash[2]) {
const prof_bt_node_t *bt_node = (prof_bt_node_t *)key;
prof_bt_hash((void *)(&bt_node->bt), r_hash);
}
static bool
prof_bt_node_keycomp(const void *k1, const void *k2) {
const prof_bt_node_t *bt_node1 = (prof_bt_node_t *)k1;
const prof_bt_node_t *bt_node2 = (prof_bt_node_t *)k2;
return prof_bt_keycomp((void *)(&bt_node1->bt),
(void *)(&bt_node2->bt));
}
static void
prof_thr_node_hash(const void *key, size_t r_hash[2]) {
const prof_thr_node_t *thr_node = (prof_thr_node_t *)key;
hash(&thr_node->thr_uid, sizeof(uint64_t), 0x94122f35U, r_hash);
}
static bool
prof_thr_node_keycomp(const void *k1, const void *k2) {
const prof_thr_node_t *thr_node1 = (prof_thr_node_t *)k1;
const prof_thr_node_t *thr_node2 = (prof_thr_node_t *)k2;
return thr_node1->thr_uid == thr_node2->thr_uid;
}
static uint64_t
prof_thr_uid_alloc(tsdn_t *tsdn) {
uint64_t thr_uid;
malloc_mutex_lock(tsdn, &next_thr_uid_mtx);
thr_uid = next_thr_uid;
next_thr_uid++;
malloc_mutex_unlock(tsdn, &next_thr_uid_mtx);
return thr_uid;
}
static prof_tdata_t *
prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
char *thread_name, bool active) {
prof_tdata_t *tdata;
cassert(config_prof);
/* Initialize an empty cache for this thread. */
tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),
sz_size2index(sizeof(prof_tdata_t)), false, NULL, true,
arena_get(TSDN_NULL, 0, true), true);
if (tdata == NULL) {
return NULL;
}
tdata->lock = prof_tdata_mutex_choose(thr_uid);
tdata->thr_uid = thr_uid;
tdata->thr_discrim = thr_discrim;
tdata->thread_name = thread_name;
tdata->attached = true;
tdata->expired = false;
tdata->tctx_uid_next = 0;
if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,
prof_bt_keycomp)) {
idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
return NULL;
}
tdata->prng_state = (uint64_t)(uintptr_t)tdata;
prof_sample_threshold_update(tdata);
tdata->enq = false;
tdata->enq_idump = false;
tdata->enq_gdump = false;
tdata->dumping = false;
tdata->active = active;
malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
tdata_tree_insert(&tdatas, tdata);
malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
return tdata;
}
prof_tdata_t *
prof_tdata_init(tsd_t *tsd) {
return prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd_tsdn(tsd)), 0,
NULL, prof_thread_active_init_get(tsd_tsdn(tsd)));
}
static bool
prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) {
if (tdata->attached && !even_if_attached) {
return false;
}
if (ckh_count(&tdata->bt2tctx) != 0) {
return false;
}
return true;
}
static bool
prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
bool even_if_attached) {
malloc_mutex_assert_owner(tsdn, tdata->lock);
return prof_tdata_should_destroy_unlocked(tdata, even_if_attached);
}
static void
prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
bool even_if_attached) {
malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx);
tdata_tree_remove(&tdatas, tdata);
assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
if (tdata->thread_name != NULL) {
idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
true);
}
ckh_delete(tsd, &tdata->bt2tctx);
idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
}
static void
prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) {
malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
}
static void
prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) {
bool destroy_tdata;
malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
if (tdata->attached) {
destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,
true);
/*
* Only detach if !destroy_tdata, because detaching would allow
* another thread to win the race to destroy tdata.
*/
if (!destroy_tdata) {
tdata->attached = false;
}
tsd_prof_tdata_set(tsd, NULL);
} else {
destroy_tdata = false;
}
malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
if (destroy_tdata) {
prof_tdata_destroy(tsd, tdata, true);
}
}
prof_tdata_t *
prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) {
uint64_t thr_uid = tdata->thr_uid;
uint64_t thr_discrim = tdata->thr_discrim + 1;
char *thread_name = (tdata->thread_name != NULL) ?
prof_thread_name_alloc(tsd_tsdn(tsd), tdata->thread_name) : NULL;
bool active = tdata->active;
prof_tdata_detach(tsd, tdata);
return prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name,
active);
}
static bool
prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) {
bool destroy_tdata;
malloc_mutex_lock(tsdn, tdata->lock);
if (!tdata->expired) {
tdata->expired = true;
destroy_tdata = tdata->attached ? false :
prof_tdata_should_destroy(tsdn, tdata, false);
} else {
destroy_tdata = false;
}
malloc_mutex_unlock(tsdn, tdata->lock);
return destroy_tdata;
}
static prof_tdata_t *
prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
void *arg) {
tsdn_t *tsdn = (tsdn_t *)arg;
return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL);
}
void
prof_reset(tsd_t *tsd, size_t lg_sample) {
prof_tdata_t *next;
assert(lg_sample < (sizeof(uint64_t) << 3));
malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
lg_prof_sample = lg_sample;
next = NULL;
do {
prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
prof_tdata_reset_iter, (void *)tsd);
if (to_destroy != NULL) {
next = tdata_tree_next(&tdatas, to_destroy);
prof_tdata_destroy_locked(tsd, to_destroy, false);
} else {
next = NULL;
}
} while (next != NULL);
malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
}
void
prof_tdata_cleanup(tsd_t *tsd) {
prof_tdata_t *tdata;
if (!config_prof) {
return;
}
tdata = tsd_prof_tdata_get(tsd);
if (tdata != NULL) {
prof_tdata_detach(tsd, tdata);
}
}
bool
prof_active_get(tsdn_t *tsdn) {
bool prof_active_current;
malloc_mutex_lock(tsdn, &prof_active_mtx);
prof_active_current = prof_active;
malloc_mutex_unlock(tsdn, &prof_active_mtx);
return prof_active_current;
}
bool
prof_active_set(tsdn_t *tsdn, bool active) {
bool prof_active_old;
malloc_mutex_lock(tsdn, &prof_active_mtx);
prof_active_old = prof_active;
prof_active = active;
malloc_mutex_unlock(tsdn, &prof_active_mtx);
return prof_active_old;
}
#ifdef JEMALLOC_JET
size_t
prof_log_bt_count(void) {
size_t cnt = 0;
prof_bt_node_t *node = log_bt_first;
while (node != NULL) {
cnt++;
node = node->next;
}
return cnt;
}
size_t
prof_log_alloc_count(void) {
size_t cnt = 0;
prof_alloc_node_t *node = log_alloc_first;
while (node != NULL) {
cnt++;
node = node->next;
}
return cnt;
}
size_t
prof_log_thr_count(void) {
size_t cnt = 0;
prof_thr_node_t *node = log_thr_first;
while (node != NULL) {
cnt++;
node = node->next;
}
return cnt;
}
bool
prof_log_is_logging(void) {
return prof_logging_state == prof_logging_state_started;
}
bool
prof_log_rep_check(void) {
if (prof_logging_state == prof_logging_state_stopped
&& log_tables_initialized) {
return true;
}
if (log_bt_last != NULL && log_bt_last->next != NULL) {
return true;
}
if (log_thr_last != NULL && log_thr_last->next != NULL) {
return true;
}
if (log_alloc_last != NULL && log_alloc_last->next != NULL) {
return true;
}
size_t bt_count = prof_log_bt_count();
size_t thr_count = prof_log_thr_count();
size_t alloc_count = prof_log_alloc_count();
if (prof_logging_state == prof_logging_state_stopped) {
if (bt_count != 0 || thr_count != 0 || alloc_count || 0) {
return true;
}
}
prof_alloc_node_t *node = log_alloc_first;
while (node != NULL) {
if (node->alloc_bt_ind >= bt_count) {
return true;
}
if (node->free_bt_ind >= bt_count) {
return true;
}
if (node->alloc_thr_ind >= thr_count) {
return true;
}
if (node->free_thr_ind >= thr_count) {
return true;
}
if (node->alloc_time_ns > node->free_time_ns) {
return true;
}
node = node->next;
}
return false;
return prof_sample_new_event_wait(tsd);
}
void
prof_log_dummy_set(bool new_value) {
prof_log_dummy = new_value;
}
#endif
bool
prof_log_start(tsdn_t *tsdn, const char *filename) {
if (!opt_prof || !prof_booted) {
return true;
}
bool ret = false;
size_t buf_size = PATH_MAX + 1;
malloc_mutex_lock(tsdn, &log_mtx);
if (prof_logging_state != prof_logging_state_stopped) {
ret = true;
} else if (filename == NULL) {
/* Make default name. */
malloc_snprintf(log_filename, buf_size, "%s.%d.%"FMTu64".json",
opt_prof_prefix, prof_getpid(), log_seq);
log_seq++;
prof_logging_state = prof_logging_state_started;
} else if (strlen(filename) >= buf_size) {
ret = true;
} else {
strcpy(log_filename, filename);
prof_logging_state = prof_logging_state_started;
prof_sample_event_handler(tsd_t *tsd, uint64_t elapsed) {
cassert(config_prof);
assert(elapsed > 0 && elapsed != TE_INVALID_ELAPSED);
if (prof_interval == 0 || !prof_active_get_unlocked()) {
return;
}
if (!ret) {
nstime_update(&log_start_timestamp);
if (counter_accum(tsd_tsdn(tsd), &prof_idump_accumulated, elapsed)) {
prof_idump(tsd_tsdn(tsd));
}
malloc_mutex_unlock(tsdn, &log_mtx);
return ret;
}
/* Used as an atexit function to stop logging on exit. */
static void
prof_log_stop_final(void) {
tsd_t *tsd = tsd_fetch();
prof_log_stop(tsd_tsdn(tsd));
}
prof_fdump(void) {
tsd_t *tsd;
struct prof_emitter_cb_arg_s {
int fd;
ssize_t ret;
};
cassert(config_prof);
assert(opt_prof_final);
static void
prof_emitter_write_cb(void *opaque, const char *to_write) {
struct prof_emitter_cb_arg_s *arg =
(struct prof_emitter_cb_arg_s *)opaque;
size_t bytes = strlen(to_write);
#ifdef JEMALLOC_JET
if (prof_log_dummy) {
if (!prof_booted) {
return;
}
#endif
arg->ret = write(arg->fd, (void *)to_write, bytes);
tsd = tsd_fetch();
assert(tsd_reentrancy_level_get(tsd) == 0);
prof_fdump_impl(tsd);
}
/*
* prof_log_emit_{...} goes through the appropriate linked list, emitting each
* node to the json and deallocating it.
*/
static void
prof_log_emit_threads(tsd_t *tsd, emitter_t *emitter) {
emitter_json_array_kv_begin(emitter, "threads");
prof_thr_node_t *thr_node = log_thr_first;
prof_thr_node_t *thr_old_node;
while (thr_node != NULL) {
emitter_json_object_begin(emitter);
static bool
prof_idump_accum_init(void) {
cassert(config_prof);
emitter_json_kv(emitter, "thr_uid", emitter_type_uint64,
&thr_node->thr_uid);
return counter_accum_init(&prof_idump_accumulated, prof_interval);
}
char *thr_name = thr_node->name;
void
prof_idump(tsdn_t *tsdn) {
tsd_t *tsd;
prof_tdata_t *tdata;
emitter_json_kv(emitter, "thr_name", emitter_type_string,
&thr_name);
cassert(config_prof);
emitter_json_object_end(emitter);
thr_old_node = thr_node;
thr_node = thr_node->next;
idalloc(tsd, thr_old_node);
if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) {
return;
}
emitter_json_array_end(emitter);
}
static void
prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) {
emitter_json_array_kv_begin(emitter, "stack_traces");
prof_bt_node_t *bt_node = log_bt_first;
prof_bt_node_t *bt_old_node;
/*
* Calculate how many hex digits we need: twice number of bytes, two for
* "0x", and then one more for terminating '\0'.
*/
char buf[2 * sizeof(intptr_t) + 3];
size_t buf_sz = sizeof(buf);
while (bt_node != NULL) {
emitter_json_array_begin(emitter);
size_t i;
for (i = 0; i < bt_node->bt.len; i++) {
malloc_snprintf(buf, buf_sz, "%p", bt_node->bt.vec[i]);
char *trace_str = buf;
emitter_json_value(emitter, emitter_type_string,
&trace_str);
}
emitter_json_array_end(emitter);
bt_old_node = bt_node;
bt_node = bt_node->next;
idalloc(tsd, bt_old_node);
tsd = tsdn_tsd(tsdn);
if (tsd_reentrancy_level_get(tsd) > 0) {
return;
}
emitter_json_array_end(emitter);
}
static void
prof_log_emit_allocs(tsd_t *tsd, emitter_t *emitter) {
emitter_json_array_kv_begin(emitter, "allocations");
prof_alloc_node_t *alloc_node = log_alloc_first;
prof_alloc_node_t *alloc_old_node;
while (alloc_node != NULL) {
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "alloc_thread", emitter_type_size,
&alloc_node->alloc_thr_ind);
tdata = prof_tdata_get(tsd, true);
if (tdata == NULL) {
return;
}
if (tdata->enq) {
tdata->enq_idump = true;
return;
}
emitter_json_kv(emitter, "free_thread", emitter_type_size,
&alloc_node->free_thr_ind);
prof_idump_impl(tsd);
}
emitter_json_kv(emitter, "alloc_trace", emitter_type_size,
&alloc_node->alloc_bt_ind);
bool
prof_mdump(tsd_t *tsd, const char *filename) {
cassert(config_prof);
assert(tsd_reentrancy_level_get(tsd) == 0);
emitter_json_kv(emitter, "free_trace", emitter_type_size,
&alloc_node->free_bt_ind);
if (!opt_prof || !prof_booted) {
return true;
}
emitter_json_kv(emitter, "alloc_timestamp",
emitter_type_uint64, &alloc_node->alloc_time_ns);
return prof_mdump_impl(tsd, filename);
}
emitter_json_kv(emitter, "free_timestamp", emitter_type_uint64,
&alloc_node->free_time_ns);
void
prof_gdump(tsdn_t *tsdn) {
tsd_t *tsd;
prof_tdata_t *tdata;
emitter_json_kv(emitter, "usize", emitter_type_uint64,
&alloc_node->usize);
cassert(config_prof);
emitter_json_object_end(emitter);
if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) {
return;
}
tsd = tsdn_tsd(tsdn);
if (tsd_reentrancy_level_get(tsd) > 0) {
return;
}
alloc_old_node = alloc_node;
alloc_node = alloc_node->next;
idalloc(tsd, alloc_old_node);
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL) {
return;
}
if (tdata->enq) {
tdata->enq_gdump = true;
return;
}
emitter_json_array_end(emitter);
}
static void
prof_log_emit_metadata(emitter_t *emitter) {
emitter_json_object_kv_begin(emitter, "info");
prof_gdump_impl(tsd);
}
nstime_t now = NSTIME_ZERO_INITIALIZER;
static uint64_t
prof_thr_uid_alloc(tsdn_t *tsdn) {
uint64_t thr_uid;
nstime_update(&now);
uint64_t ns = nstime_ns(&now) - nstime_ns(&log_start_timestamp);
emitter_json_kv(emitter, "duration", emitter_type_uint64, &ns);
malloc_mutex_lock(tsdn, &next_thr_uid_mtx);
thr_uid = next_thr_uid;
next_thr_uid++;
malloc_mutex_unlock(tsdn, &next_thr_uid_mtx);
char *vers = JEMALLOC_VERSION;
emitter_json_kv(emitter, "version",
emitter_type_string, &vers);
return thr_uid;
}
emitter_json_kv(emitter, "lg_sample_rate",
emitter_type_int, &lg_prof_sample);
prof_tdata_t *
prof_tdata_init(tsd_t *tsd) {
return prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd_tsdn(tsd)), 0,
NULL, prof_thread_active_init_get(tsd_tsdn(tsd)));
}
int pid = prof_getpid();
emitter_json_kv(emitter, "pid", emitter_type_int, &pid);
prof_tdata_t *
prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) {
uint64_t thr_uid = tdata->thr_uid;
uint64_t thr_discrim = tdata->thr_discrim + 1;
char *thread_name = (tdata->thread_name != NULL) ?
prof_thread_name_alloc(tsd, tdata->thread_name) : NULL;
bool active = tdata->active;
emitter_json_object_end(emitter);
prof_tdata_detach(tsd, tdata);
return prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name,
active);
}
void
prof_tdata_cleanup(tsd_t *tsd) {
prof_tdata_t *tdata;
bool
prof_log_stop(tsdn_t *tsdn) {
if (!opt_prof || !prof_booted) {
return true;
if (!config_prof) {
return;
}
tsd_t *tsd = tsdn_tsd(tsdn);
malloc_mutex_lock(tsdn, &log_mtx);
if (prof_logging_state != prof_logging_state_started) {
malloc_mutex_unlock(tsdn, &log_mtx);
return true;
tdata = tsd_prof_tdata_get(tsd);
if (tdata != NULL) {
prof_tdata_detach(tsd, tdata);
}
}
/*
* Set the state to dumping. We'll set it to stopped when we're done.
* Since other threads won't be able to start/stop/log when the state is
* dumping, we don't have to hold the lock during the whole method.
*/
prof_logging_state = prof_logging_state_dumping;
malloc_mutex_unlock(tsdn, &log_mtx);
emitter_t emitter;
/* Create a file. */
bool
prof_active_get(tsdn_t *tsdn) {
bool prof_active_current;
int fd;
#ifdef JEMALLOC_JET
if (prof_log_dummy) {
fd = 0;
} else {
fd = creat(log_filename, 0644);
}
#else
fd = creat(log_filename, 0644);
#endif
prof_active_assert();
malloc_mutex_lock(tsdn, &prof_active_mtx);
prof_active_current = prof_active_state;
malloc_mutex_unlock(tsdn, &prof_active_mtx);
return prof_active_current;
}
if (fd == -1) {
malloc_printf("<jemalloc>: creat() for log file \"%s\" "
" failed with %d\n", log_filename, errno);
if (opt_abort) {
abort();
}
return true;
}
bool
prof_active_set(tsdn_t *tsdn, bool active) {
bool prof_active_old;
/* Emit to json. */
struct prof_emitter_cb_arg_s arg;
arg.fd = fd;
emitter_init(&emitter, emitter_output_json, &prof_emitter_write_cb,
(void *)(&arg));
emitter_begin(&emitter);
prof_log_emit_metadata(&emitter);
prof_log_emit_threads(tsd, &emitter);
prof_log_emit_traces(tsd, &emitter);
prof_log_emit_allocs(tsd, &emitter);
emitter_end(&emitter);
/* Reset global state. */
if (log_tables_initialized) {
ckh_delete(tsd, &log_bt_node_set);
ckh_delete(tsd, &log_thr_node_set);
}
log_tables_initialized = false;
log_bt_index = 0;
log_thr_index = 0;
log_bt_first = NULL;
log_bt_last = NULL;
log_thr_first = NULL;
log_thr_last = NULL;
log_alloc_first = NULL;
log_alloc_last = NULL;
malloc_mutex_lock(tsdn, &log_mtx);
prof_logging_state = prof_logging_state_stopped;
malloc_mutex_unlock(tsdn, &log_mtx);
#ifdef JEMALLOC_JET
if (prof_log_dummy) {
return false;
}
#endif
return close(fd);
prof_active_assert();
malloc_mutex_lock(tsdn, &prof_active_mtx);
prof_active_old = prof_active_state;
prof_active_state = active;
malloc_mutex_unlock(tsdn, &prof_active_mtx);
prof_active_assert();
return prof_active_old;
}
const char *
prof_thread_name_get(tsd_t *tsd) {
assert(tsd_reentrancy_level_get(tsd) == 0);
prof_tdata_t *tdata;
tdata = prof_tdata_get(tsd, true);
......@@ -2790,69 +448,19 @@ prof_thread_name_get(tsd_t *tsd) {
return (tdata->thread_name != NULL ? tdata->thread_name : "");
}
static char *
prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name) {
char *ret;
size_t size;
if (thread_name == NULL) {
return NULL;
}
size = strlen(thread_name) + 1;
if (size == 1) {
return "";
}
ret = iallocztm(tsdn, size, sz_size2index(size), false, NULL, true,
arena_get(TSDN_NULL, 0, true), true);
if (ret == NULL) {
return NULL;
}
memcpy(ret, thread_name, size);
return ret;
}
int
prof_thread_name_set(tsd_t *tsd, const char *thread_name) {
prof_tdata_t *tdata;
unsigned i;
char *s;
tdata = prof_tdata_get(tsd, true);
if (tdata == NULL) {
return EAGAIN;
}
/* Validate input. */
if (thread_name == NULL) {
return EFAULT;
}
for (i = 0; thread_name[i] != '\0'; i++) {
char c = thread_name[i];
if (!isgraph(c) && !isblank(c)) {
return EFAULT;
}
}
s = prof_thread_name_alloc(tsd_tsdn(tsd), thread_name);
if (s == NULL) {
return EAGAIN;
}
if (tdata->thread_name != NULL) {
idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
true);
tdata->thread_name = NULL;
}
if (strlen(s) > 0) {
tdata->thread_name = s;
if (opt_prof_sys_thread_name) {
return ENOENT;
} else {
return prof_thread_name_set_impl(tsd, thread_name);
}
return 0;
}
bool
prof_thread_active_get(tsd_t *tsd) {
assert(tsd_reentrancy_level_get(tsd) == 0);
prof_tdata_t *tdata;
tdata = prof_tdata_get(tsd, true);
......@@ -2864,6 +472,8 @@ prof_thread_active_get(tsd_t *tsd) {
bool
prof_thread_active_set(tsd_t *tsd, bool active) {
assert(tsd_reentrancy_level_get(tsd) == 0);
prof_tdata_t *tdata;
tdata = prof_tdata_get(tsd, true);
......@@ -2916,6 +526,28 @@ prof_gdump_set(tsdn_t *tsdn, bool gdump) {
return prof_gdump_old;
}
void
prof_backtrace_hook_set(prof_backtrace_hook_t hook) {
atomic_store_p(&prof_backtrace_hook, hook, ATOMIC_RELEASE);
}
prof_backtrace_hook_t
prof_backtrace_hook_get() {
return (prof_backtrace_hook_t)atomic_load_p(&prof_backtrace_hook,
ATOMIC_ACQUIRE);
}
void
prof_dump_hook_set(prof_dump_hook_t hook) {
atomic_store_p(&prof_dump_hook, hook, ATOMIC_RELEASE);
}
prof_dump_hook_t
prof_dump_hook_get() {
return (prof_dump_hook_t)atomic_load_p(&prof_dump_hook,
ATOMIC_ACQUIRE);
}
void
prof_boot0(void) {
cassert(config_prof);
......@@ -2932,6 +564,9 @@ prof_boot1(void) {
* opt_prof must be in its final state before any arenas are
* initialized, so this function must be executed early.
*/
if (opt_prof_leak_error && !opt_prof_leak) {
opt_prof_leak = true;
}
if (opt_prof_leak && !opt_prof) {
/*
......@@ -2949,61 +584,65 @@ prof_boot1(void) {
}
bool
prof_boot2(tsd_t *tsd) {
prof_boot2(tsd_t *tsd, base_t *base) {
cassert(config_prof);
if (opt_prof) {
unsigned i;
/*
* Initialize the global mutexes unconditionally to maintain correct
* stats when opt_prof is false.
*/
if (malloc_mutex_init(&prof_active_mtx, "prof_active",
WITNESS_RANK_PROF_ACTIVE, malloc_mutex_rank_exclusive)) {
return true;
}
if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump",
WITNESS_RANK_PROF_GDUMP, malloc_mutex_rank_exclusive)) {
return true;
}
if (malloc_mutex_init(&prof_thread_active_init_mtx,
"prof_thread_active_init", WITNESS_RANK_PROF_THREAD_ACTIVE_INIT,
malloc_mutex_rank_exclusive)) {
return true;
}
if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx",
WITNESS_RANK_PROF_BT2GCTX, malloc_mutex_rank_exclusive)) {
return true;
}
if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas",
WITNESS_RANK_PROF_TDATAS, malloc_mutex_rank_exclusive)) {
return true;
}
if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid",
WITNESS_RANK_PROF_NEXT_THR_UID, malloc_mutex_rank_exclusive)) {
return true;
}
if (malloc_mutex_init(&prof_stats_mtx, "prof_stats",
WITNESS_RANK_PROF_STATS, malloc_mutex_rank_exclusive)) {
return true;
}
if (malloc_mutex_init(&prof_dump_filename_mtx,
"prof_dump_filename", WITNESS_RANK_PROF_DUMP_FILENAME,
malloc_mutex_rank_exclusive)) {
return true;
}
if (malloc_mutex_init(&prof_dump_mtx, "prof_dump",
WITNESS_RANK_PROF_DUMP, malloc_mutex_rank_exclusive)) {
return true;
}
if (opt_prof) {
lg_prof_sample = opt_lg_prof_sample;
prof_active = opt_prof_active;
if (malloc_mutex_init(&prof_active_mtx, "prof_active",
WITNESS_RANK_PROF_ACTIVE, malloc_mutex_rank_exclusive)) {
return true;
}
prof_unbias_map_init();
prof_active_state = opt_prof_active;
prof_gdump_val = opt_prof_gdump;
if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump",
WITNESS_RANK_PROF_GDUMP, malloc_mutex_rank_exclusive)) {
return true;
}
prof_thread_active_init = opt_prof_thread_active_init;
if (malloc_mutex_init(&prof_thread_active_init_mtx,
"prof_thread_active_init",
WITNESS_RANK_PROF_THREAD_ACTIVE_INIT,
malloc_mutex_rank_exclusive)) {
return true;
}
if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
prof_bt_keycomp)) {
return true;
}
if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx",
WITNESS_RANK_PROF_BT2GCTX, malloc_mutex_rank_exclusive)) {
return true;
}
tdata_tree_new(&tdatas);
if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas",
WITNESS_RANK_PROF_TDATAS, malloc_mutex_rank_exclusive)) {
if (prof_data_init(tsd)) {
return true;
}
next_thr_uid = 0;
if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid",
WITNESS_RANK_PROF_NEXT_THR_UID, malloc_mutex_rank_exclusive)) {
return true;
}
if (malloc_mutex_init(&prof_dump_seq_mtx, "prof_dump_seq",
WITNESS_RANK_PROF_DUMP_SEQ, malloc_mutex_rank_exclusive)) {
return true;
}
if (malloc_mutex_init(&prof_dump_mtx, "prof_dump",
WITNESS_RANK_PROF_DUMP, malloc_mutex_rank_exclusive)) {
if (prof_idump_accum_init()) {
return true;
}
......@@ -3015,42 +654,22 @@ prof_boot2(tsd_t *tsd) {
}
}
if (opt_prof_log) {
prof_log_start(tsd_tsdn(tsd), NULL);
}
if (atexit(prof_log_stop_final) != 0) {
malloc_write("<jemalloc>: Error in atexit() "
"for logging\n");
if (opt_abort) {
abort();
}
}
if (malloc_mutex_init(&log_mtx, "prof_log",
WITNESS_RANK_PROF_LOG, malloc_mutex_rank_exclusive)) {
return true;
}
if (ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS,
prof_bt_node_hash, prof_bt_node_keycomp)) {
if (prof_log_init(tsd)) {
return true;
}
if (ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS,
prof_thr_node_hash, prof_thr_node_keycomp)) {
if (prof_recent_init()) {
return true;
}
log_tables_initialized = true;
prof_base = base;
gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),
b0get(), PROF_NCTX_LOCKS * sizeof(malloc_mutex_t),
CACHELINE);
gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), base,
PROF_NCTX_LOCKS * sizeof(malloc_mutex_t), CACHELINE);
if (gctx_locks == NULL) {
return true;
}
for (i = 0; i < PROF_NCTX_LOCKS; i++) {
for (unsigned i = 0; i < PROF_NCTX_LOCKS; i++) {
if (malloc_mutex_init(&gctx_locks[i], "prof_gctx",
WITNESS_RANK_PROF_GCTX,
malloc_mutex_rank_exclusive)) {
......@@ -3058,26 +677,21 @@ prof_boot2(tsd_t *tsd) {
}
}
tdata_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),
b0get(), PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t),
CACHELINE);
tdata_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), base,
PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t), CACHELINE);
if (tdata_locks == NULL) {
return true;
}
for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
for (unsigned i = 0; i < PROF_NTDATA_LOCKS; i++) {
if (malloc_mutex_init(&tdata_locks[i], "prof_tdata",
WITNESS_RANK_PROF_TDATA,
malloc_mutex_rank_exclusive)) {
return true;
}
}
#ifdef JEMALLOC_PROF_LIBGCC
/*
* Cause the backtracing machinery to allocate its internal
* state before enabling profiling.
*/
_Unwind_Backtrace(prof_unwind_init_callback, NULL);
#endif
prof_unwind_init();
prof_hooks_init();
}
prof_booted = true;
......@@ -3095,18 +709,23 @@ prof_prefork0(tsdn_t *tsdn) {
for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
malloc_mutex_prefork(tsdn, &tdata_locks[i]);
}
malloc_mutex_prefork(tsdn, &log_mtx);
for (i = 0; i < PROF_NCTX_LOCKS; i++) {
malloc_mutex_prefork(tsdn, &gctx_locks[i]);
}
malloc_mutex_prefork(tsdn, &prof_recent_dump_mtx);
}
}
void
prof_prefork1(tsdn_t *tsdn) {
if (config_prof && opt_prof) {
counter_prefork(tsdn, &prof_idump_accumulated);
malloc_mutex_prefork(tsdn, &prof_active_mtx);
malloc_mutex_prefork(tsdn, &prof_dump_seq_mtx);
malloc_mutex_prefork(tsdn, &prof_dump_filename_mtx);
malloc_mutex_prefork(tsdn, &prof_gdump_mtx);
malloc_mutex_prefork(tsdn, &prof_recent_alloc_mtx);
malloc_mutex_prefork(tsdn, &prof_stats_mtx);
malloc_mutex_prefork(tsdn, &next_thr_uid_mtx);
malloc_mutex_prefork(tsdn, &prof_thread_active_init_mtx);
}
......@@ -3120,12 +739,17 @@ prof_postfork_parent(tsdn_t *tsdn) {
malloc_mutex_postfork_parent(tsdn,
&prof_thread_active_init_mtx);
malloc_mutex_postfork_parent(tsdn, &next_thr_uid_mtx);
malloc_mutex_postfork_parent(tsdn, &prof_stats_mtx);
malloc_mutex_postfork_parent(tsdn, &prof_recent_alloc_mtx);
malloc_mutex_postfork_parent(tsdn, &prof_gdump_mtx);
malloc_mutex_postfork_parent(tsdn, &prof_dump_seq_mtx);
malloc_mutex_postfork_parent(tsdn, &prof_dump_filename_mtx);
malloc_mutex_postfork_parent(tsdn, &prof_active_mtx);
counter_postfork_parent(tsdn, &prof_idump_accumulated);
malloc_mutex_postfork_parent(tsdn, &prof_recent_dump_mtx);
for (i = 0; i < PROF_NCTX_LOCKS; i++) {
malloc_mutex_postfork_parent(tsdn, &gctx_locks[i]);
}
malloc_mutex_postfork_parent(tsdn, &log_mtx);
for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
malloc_mutex_postfork_parent(tsdn, &tdata_locks[i]);
}
......@@ -3142,12 +766,17 @@ prof_postfork_child(tsdn_t *tsdn) {
malloc_mutex_postfork_child(tsdn, &prof_thread_active_init_mtx);
malloc_mutex_postfork_child(tsdn, &next_thr_uid_mtx);
malloc_mutex_postfork_child(tsdn, &prof_stats_mtx);
malloc_mutex_postfork_child(tsdn, &prof_recent_alloc_mtx);
malloc_mutex_postfork_child(tsdn, &prof_gdump_mtx);
malloc_mutex_postfork_child(tsdn, &prof_dump_seq_mtx);
malloc_mutex_postfork_child(tsdn, &prof_dump_filename_mtx);
malloc_mutex_postfork_child(tsdn, &prof_active_mtx);
counter_postfork_child(tsdn, &prof_idump_accumulated);
malloc_mutex_postfork_child(tsdn, &prof_recent_dump_mtx);
for (i = 0; i < PROF_NCTX_LOCKS; i++) {
malloc_mutex_postfork_child(tsdn, &gctx_locks[i]);
}
malloc_mutex_postfork_child(tsdn, &log_mtx);
for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
malloc_mutex_postfork_child(tsdn, &tdata_locks[i]);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment