Commit b8beda3c authored by Oran Agra's avatar Oran Agra
Browse files

Merge commit jemalloc 5.3.0

parents d659c734 6d23d3ac
...@@ -31,7 +31,6 @@ install: ...@@ -31,7 +31,6 @@ install:
- set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH% - set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH%
- if defined MSVC call "c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %MSVC% - if defined MSVC call "c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %MSVC%
- if defined MSVC pacman --noconfirm -Rsc mingw-w64-%CPU%-gcc gcc - if defined MSVC pacman --noconfirm -Rsc mingw-w64-%CPU%-gcc gcc
- pacman --noconfirm -Suy mingw-w64-%CPU%-make
build_script: build_script:
- bash -c "autoconf" - bash -c "autoconf"
......
...@@ -3,18 +3,43 @@ env: ...@@ -3,18 +3,43 @@ env:
ARCH: amd64 ARCH: amd64
task: task:
matrix:
env:
DEBUG_CONFIG: --enable-debug
env:
DEBUG_CONFIG: --disable-debug
matrix:
- env:
PROF_CONFIG: --enable-prof
- env:
PROF_CONFIG: --disable-prof
matrix:
- name: 64-bit
env:
CC:
CXX:
- name: 32-bit
env:
CC: cc -m32
CXX: c++ -m32
matrix:
- env:
UNCOMMON_CONFIG:
- env:
UNCOMMON_CONFIG: --with-lg-page=16 --with-malloc-conf=tcache:false
freebsd_instance: freebsd_instance:
matrix: matrix:
image: freebsd-12-0-release-amd64 image: freebsd-12-3-release-amd64
image: freebsd-11-2-release-amd64
install_script: install_script:
- sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf - sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf
- pkg upgrade -y - pkg upgrade -y
- pkg install -y autoconf gmake - pkg install -y autoconf gmake
script: script:
- autoconf - autoconf
#- ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" CXX="$CXX $COMPILER_FLAGS" } $CONFIGURE_FLAGS # We don't perfectly track freebsd stdlib.h definitions. This is fine when
- ./configure # we count as a system header, but breaks otherwise, like during these
# tests.
- ./configure --with-jemalloc-prefix=ci_ ${DEBUG_CONFIG} ${PROF_CONFIG} ${UNCOMMON_CONFIG}
- export JFLAG=`sysctl -n kern.smp.cpus` - export JFLAG=`sysctl -n kern.smp.cpus`
- gmake -j${JFLAG} - gmake -j${JFLAG}
- gmake -j${JFLAG} tests - gmake -j${JFLAG} tests
......
# jemalloc targets clang-format version 8. We include every option it supports
# here, but comment out the ones that aren't relevant for us.
---
# AccessModifierOffset: -2
AlignAfterOpenBracket: DontAlign
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: false
AlignEscapedNewlines: Right
AlignOperands: false
AlignTrailingComments: false
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Empty
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterReturnType: AllDefinitions
AlwaysBreakBeforeMultilineStrings: true
# AlwaysBreakTemplateDeclarations: Yes
BinPackArguments: true
BinPackParameters: true
BraceWrapping:
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: false
AfterNamespace: false
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
# BreakAfterJavaFieldAnnotations: true
BreakBeforeBinaryOperators: NonAssignment
BreakBeforeBraces: Attach
BreakBeforeTernaryOperators: true
# BreakConstructorInitializers: BeforeColon
# BreakInheritanceList: BeforeColon
BreakStringLiterals: false
ColumnLimit: 80
# CommentPragmas: ''
# CompactNamespaces: true
# ConstructorInitializerAllOnOneLineOrOnePerLine: true
# ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 2
Cpp11BracedListStyle: true
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
ForEachMacros: [ ql_foreach, qr_foreach, ]
# IncludeBlocks: Preserve
# IncludeCategories:
# - Regex: '^<.*\.h(pp)?>'
# Priority: 1
# IncludeIsMainRegex: ''
IndentCaseLabels: false
IndentPPDirectives: AfterHash
IndentWidth: 4
IndentWrappedFunctionNames: false
# JavaImportGroups: []
# JavaScriptQuotes: Leave
# JavaScriptWrapImports: True
KeepEmptyLinesAtTheStartOfBlocks: false
Language: Cpp
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 1
# NamespaceIndentation: None
# ObjCBinPackProtocolList: Auto
# ObjCBlockIndentWidth: 2
# ObjCSpaceAfterProperty: false
# ObjCSpaceBeforeProtocolList: false
PenaltyBreakAssignment: 2
PenaltyBreakBeforeFirstCallParameter: 1
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
# PenaltyBreakTemplateDeclaration: 10
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 60
PointerAlignment: Right
# RawStringFormats:
# - Language: TextProto
# Delimiters:
# - 'pb'
# - 'proto'
# EnclosingFunctions:
# - 'PARSE_TEXT_PROTO'
# BasedOnStyle: google
# - Language: Cpp
# Delimiters:
# - 'cc'
# - 'cpp'
# BasedOnStyle: llvm
# CanonicalDelimiter: 'cc'
ReflowComments: true
SortIncludes: false
SpaceAfterCStyleCast: false
# SpaceAfterTemplateKeyword: true
SpaceBeforeAssignmentOperators: true
# SpaceBeforeCpp11BracedList: false
# SpaceBeforeCtorInitializerColon: true
# SpaceBeforeInheritanceColon: true
SpaceBeforeParens: ControlStatements
# SpaceBeforeRangeBasedForLoopColon: true
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 2
SpacesInAngles: false
SpacesInCStyleCastParentheses: false
# SpacesInContainerLiterals: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
# Standard: Cpp11
# This is nominally supported in clang-format version 8, but not in the build
# used by some of the core jemalloc developers.
# StatementMacros: []
TabWidth: 8
UseTab: Never
...
...@@ -13,6 +13,8 @@ ...@@ -13,6 +13,8 @@
/doc/jemalloc.html /doc/jemalloc.html
/doc/jemalloc.3 /doc/jemalloc.3
/doc_internal/PROFILING_INTERNALS.pdf
/jemalloc.pc /jemalloc.pc
/lib/ /lib/
...@@ -50,6 +52,7 @@ test/include/test/jemalloc_test.h ...@@ -50,6 +52,7 @@ test/include/test/jemalloc_test.h
test/include/test/jemalloc_test_defs.h test/include/test/jemalloc_test_defs.h
/test/integration/[A-Za-z]* /test/integration/[A-Za-z]*
!/test/integration/cpp/
!/test/integration/[A-Za-z]*.* !/test/integration/[A-Za-z]*.*
/test/integration/*.[od] /test/integration/*.[od]
/test/integration/*.out /test/integration/*.out
...@@ -71,6 +74,11 @@ test/include/test/jemalloc_test_defs.h ...@@ -71,6 +74,11 @@ test/include/test/jemalloc_test_defs.h
/test/unit/*.[od] /test/unit/*.[od]
/test/unit/*.out /test/unit/*.out
/test/analyze/[A-Za-z]*
!/test/analyze/[A-Za-z]*.*
/test/analyze/*.[od]
/test/analyze/*.out
/VERSION /VERSION
*.pdb *.pdb
......
This diff is collapsed.
...@@ -4,6 +4,106 @@ brevity. Much more detail can be found in the git revision history: ...@@ -4,6 +4,106 @@ brevity. Much more detail can be found in the git revision history:
https://github.com/jemalloc/jemalloc https://github.com/jemalloc/jemalloc
* 5.3.0 (May 6, 2022)
This release contains many speed and space optimizations, from micro
optimizations on common paths to rework of internal data structures and
locking schemes, and many more too detailed to list below. Multiple percent
of system level metric improvements were measured in tested production
workloads. The release has gone through large-scale production testing.
New features:
- Add the thread.idle mallctl which hints that the calling thread will be
idle for a nontrivial period of time. (@davidtgoldblatt)
- Allow small size classes to be the maximum size class to cache in the
thread-specific cache, through the opt.[lg_]tcache_max option. (@interwq,
@jordalgo)
- Make the behavior of realloc(ptr, 0) configurable with opt.zero_realloc.
(@davidtgoldblatt)
- Add 'make uninstall' support. (@sangshuduo, @Lapenkov)
- Support C++17 over-aligned allocation. (@marksantaniello)
- Add the thread.peak mallctl for approximate per-thread peak memory tracking.
(@davidtgoldblatt)
- Add interval-based stats output opt.stats_interval. (@interwq)
- Add prof.prefix to override filename prefixes for dumps. (@zhxchen17)
- Add high resolution timestamp support for profiling. (@tyroguru)
- Add the --collapsed flag to jeprof for flamegraph generation.
(@igorwwwwwwwwwwwwwwwwwwww)
- Add the --debug-syms-by-id option to jeprof for debug symbols discovery.
(@DeannaGelbart)
- Add the opt.prof_leak_error option to exit with error code when leak is
detected using opt.prof_final. (@yunxuo)
- Add opt.cache_oblivious as an runtime alternative to config.cache_oblivious.
(@interwq)
- Add mallctl interfaces:
+ opt.zero_realloc (@davidtgoldblatt)
+ opt.cache_oblivious (@interwq)
+ opt.prof_leak_error (@yunxuo)
+ opt.stats_interval (@interwq)
+ opt.stats_interval_opts (@interwq)
+ opt.tcache_max (@interwq)
+ opt.trust_madvise (@azat)
+ prof.prefix (@zhxchen17)
+ stats.zero_reallocs (@davidtgoldblatt)
+ thread.idle (@davidtgoldblatt)
+ thread.peak.{read,reset} (@davidtgoldblatt)
Bug fixes:
- Fix the synchronization around explicit tcache creation which could cause
invalid tcache identifiers. This regression was first released in 5.0.0.
(@yoshinorim, @davidtgoldblatt)
- Fix a profiling biasing issue which could cause incorrect heap usage and
object counts. This issue existed in all previous releases with the heap
profiling feature. (@davidtgoldblatt)
- Fix the order of stats counter updating on large realloc which could cause
failed assertions. This regression was first released in 5.0.0. (@azat)
- Fix the locking on the arena destroy mallctl, which could cause concurrent
arena creations to fail. This functionality was first introduced in 5.0.0.
(@interwq)
Portability improvements:
- Remove nothrow from system function declarations on macOS and FreeBSD.
(@davidtgoldblatt, @fredemmott, @leres)
- Improve overcommit and page alignment settings on NetBSD. (@zoulasc)
- Improve CPU affinity support on BSD platforms. (@devnexen)
- Improve utrace detection and support. (@devnexen)
- Improve QEMU support with MADV_DONTNEED zeroed pages detection. (@azat)
- Add memcntl support on Solaris / illumos. (@devnexen)
- Improve CPU_SPINWAIT on ARM. (@AWSjswinney)
- Improve TSD cleanup on FreeBSD. (@Lapenkov)
- Disable percpu_arena if the CPU count cannot be reliably detected. (@azat)
- Add malloc_size(3) override support. (@devnexen)
- Add mmap VM_MAKE_TAG support. (@devnexen)
- Add support for MADV_[NO]CORE. (@devnexen)
- Add support for DragonFlyBSD. (@devnexen)
- Fix the QUANTUM setting on MIPS64. (@brooksdavis)
- Add the QUANTUM setting for ARC. (@vineetgarc)
- Add the QUANTUM setting for LoongArch. (@wangjl-uos)
- Add QNX support. (@jqian-aurora)
- Avoid atexit(3) calls unless the relevant profiling features are enabled.
(@BusyJay, @laiwei-rice, @interwq)
- Fix unknown option detection when using Clang. (@Lapenkov)
- Fix symbol conflict with musl libc. (@georgthegreat)
- Add -Wimplicit-fallthrough checks. (@nickdesaulniers)
- Add __forceinline support on MSVC. (@santagada)
- Improve FreeBSD and Windows CI support. (@Lapenkov)
- Add CI support for PPC64LE architecture. (@ezeeyahoo)
Incompatible changes:
- Maximum size class allowed in tcache (opt.[lg_]tcache_max) now has an upper
bound of 8MiB. (@interwq)
Optimizations and refactors (@davidtgoldblatt, @Lapenkov, @interwq):
- Optimize the common cases of the thread cache operations.
- Optimize internal data structures, including RB tree and pairing heap.
- Optimize the internal locking on extent management.
- Extract and refactor the internal page allocator and interface modules.
Documentation:
- Fix doc build with --with-install-suffix. (@lawmurray, @interwq)
- Add PROFILING_INTERNALS.md. (@davidtgoldblatt)
- Ensure the proper order of doc building and installation. (@Mingli-Yu)
* 5.2.1 (August 5, 2019) * 5.2.1 (August 5, 2019)
This release is primarily about Windows. A critical virtual memory leak is This release is primarily about Windows. A critical virtual memory leak is
......
...@@ -9,14 +9,17 @@ If building from unpackaged developer sources, the simplest command sequence ...@@ -9,14 +9,17 @@ If building from unpackaged developer sources, the simplest command sequence
that might work is: that might work is:
./autogen.sh ./autogen.sh
make dist
make make
make install make install
Note that documentation is not built by the default target because doing so You can uninstall the installed build artifacts like this:
would create a dependency on xsltproc in packaged releases, hence the
requirement to either run 'make dist' or avoid installing docs via the various make uninstall
install_* targets documented below.
Notes:
- "autoconf" needs to be installed
- Documentation is built by the default target only when xsltproc is
available. Build will warn but not stop if the dependency is missing.
## Advanced configuration ## Advanced configuration
...@@ -188,13 +191,13 @@ any of the following arguments (not a definitive list) to 'configure': ...@@ -188,13 +191,13 @@ any of the following arguments (not a definitive list) to 'configure':
* `--disable-cache-oblivious` * `--disable-cache-oblivious`
Disable cache-oblivious large allocation alignment for large allocation Disable cache-oblivious large allocation alignment by default, for large
requests with no alignment constraints. If this feature is disabled, all allocation requests with no alignment constraints. If this feature is
large allocations are page-aligned as an implementation artifact, which can disabled, all large allocations are page-aligned as an implementation
severely harm CPU cache utilization. However, the cache-oblivious layout artifact, which can severely harm CPU cache utilization. However, the
comes at the cost of one extra page per large allocation, which in the cache-oblivious layout comes at the cost of one extra page per large
most extreme case increases physical memory usage for the 16 KiB size class allocation, which in the most extreme case increases physical memory usage
to 20 KiB. for the 16 KiB size class to 20 KiB.
* `--disable-syscall` * `--disable-syscall`
......
...@@ -99,31 +99,60 @@ C_SRCS := $(srcroot)src/jemalloc.c \ ...@@ -99,31 +99,60 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/background_thread.c \ $(srcroot)src/background_thread.c \
$(srcroot)src/base.c \ $(srcroot)src/base.c \
$(srcroot)src/bin.c \ $(srcroot)src/bin.c \
$(srcroot)src/bin_info.c \
$(srcroot)src/bitmap.c \ $(srcroot)src/bitmap.c \
$(srcroot)src/buf_writer.c \
$(srcroot)src/cache_bin.c \
$(srcroot)src/ckh.c \ $(srcroot)src/ckh.c \
$(srcroot)src/counter.c \
$(srcroot)src/ctl.c \ $(srcroot)src/ctl.c \
$(srcroot)src/decay.c \
$(srcroot)src/div.c \ $(srcroot)src/div.c \
$(srcroot)src/ecache.c \
$(srcroot)src/edata.c \
$(srcroot)src/edata_cache.c \
$(srcroot)src/ehooks.c \
$(srcroot)src/emap.c \
$(srcroot)src/eset.c \
$(srcroot)src/exp_grow.c \
$(srcroot)src/extent.c \ $(srcroot)src/extent.c \
$(srcroot)src/extent_dss.c \ $(srcroot)src/extent_dss.c \
$(srcroot)src/extent_mmap.c \ $(srcroot)src/extent_mmap.c \
$(srcroot)src/hash.c \ $(srcroot)src/fxp.c \
$(srcroot)src/san.c \
$(srcroot)src/san_bump.c \
$(srcroot)src/hook.c \ $(srcroot)src/hook.c \
$(srcroot)src/hpa.c \
$(srcroot)src/hpa_hooks.c \
$(srcroot)src/hpdata.c \
$(srcroot)src/inspect.c \
$(srcroot)src/large.c \ $(srcroot)src/large.c \
$(srcroot)src/log.c \ $(srcroot)src/log.c \
$(srcroot)src/malloc_io.c \ $(srcroot)src/malloc_io.c \
$(srcroot)src/mutex.c \ $(srcroot)src/mutex.c \
$(srcroot)src/mutex_pool.c \
$(srcroot)src/nstime.c \ $(srcroot)src/nstime.c \
$(srcroot)src/pa.c \
$(srcroot)src/pa_extra.c \
$(srcroot)src/pai.c \
$(srcroot)src/pac.c \
$(srcroot)src/pages.c \ $(srcroot)src/pages.c \
$(srcroot)src/prng.c \ $(srcroot)src/peak_event.c \
$(srcroot)src/prof.c \ $(srcroot)src/prof.c \
$(srcroot)src/prof_data.c \
$(srcroot)src/prof_log.c \
$(srcroot)src/prof_recent.c \
$(srcroot)src/prof_stats.c \
$(srcroot)src/prof_sys.c \
$(srcroot)src/psset.c \
$(srcroot)src/rtree.c \ $(srcroot)src/rtree.c \
$(srcroot)src/safety_check.c \ $(srcroot)src/safety_check.c \
$(srcroot)src/stats.c \
$(srcroot)src/sc.c \ $(srcroot)src/sc.c \
$(srcroot)src/sec.c \
$(srcroot)src/stats.c \
$(srcroot)src/sz.c \ $(srcroot)src/sz.c \
$(srcroot)src/tcache.c \ $(srcroot)src/tcache.c \
$(srcroot)src/test_hooks.c \ $(srcroot)src/test_hooks.c \
$(srcroot)src/thread_event.c \
$(srcroot)src/ticker.c \ $(srcroot)src/ticker.c \
$(srcroot)src/tsd.c \ $(srcroot)src/tsd.c \
$(srcroot)src/witness.c $(srcroot)src/witness.c
...@@ -148,88 +177,124 @@ else ...@@ -148,88 +177,124 @@ else
LJEMALLOC := $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) LJEMALLOC := $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
endif endif
PC := $(objroot)jemalloc.pc PC := $(objroot)jemalloc.pc
MAN3 := $(objroot)doc/jemalloc$(install_suffix).3
DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml
DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.html) DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.html)
DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.3) DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.3)
DOCS := $(DOCS_HTML) $(DOCS_MAN3) DOCS := $(DOCS_HTML) $(DOCS_MAN3)
C_TESTLIB_SRCS := $(srcroot)test/src/btalloc.c $(srcroot)test/src/btalloc_0.c \ C_TESTLIB_SRCS := $(srcroot)test/src/btalloc.c $(srcroot)test/src/btalloc_0.c \
$(srcroot)test/src/btalloc_1.c $(srcroot)test/src/math.c \ $(srcroot)test/src/btalloc_1.c $(srcroot)test/src/math.c \
$(srcroot)test/src/mtx.c $(srcroot)test/src/mq.c \ $(srcroot)test/src/mtx.c $(srcroot)test/src/sleep.c \
$(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \ $(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \
$(srcroot)test/src/thd.c $(srcroot)test/src/timer.c $(srcroot)test/src/thd.c $(srcroot)test/src/timer.c
ifeq (1, $(link_whole_archive)) ifeq (1, $(link_whole_archive))
C_UTIL_INTEGRATION_SRCS := C_UTIL_INTEGRATION_SRCS :=
C_UTIL_CPP_SRCS := C_UTIL_CPP_SRCS :=
else else
C_UTIL_INTEGRATION_SRCS := $(srcroot)src/nstime.c $(srcroot)src/malloc_io.c C_UTIL_INTEGRATION_SRCS := $(srcroot)src/nstime.c $(srcroot)src/malloc_io.c \
$(srcroot)src/ticker.c
C_UTIL_CPP_SRCS := $(srcroot)src/nstime.c $(srcroot)src/malloc_io.c C_UTIL_CPP_SRCS := $(srcroot)src/nstime.c $(srcroot)src/malloc_io.c
endif endif
TESTS_UNIT := \ TESTS_UNIT := \
$(srcroot)test/unit/a0.c \ $(srcroot)test/unit/a0.c \
$(srcroot)test/unit/arena_decay.c \
$(srcroot)test/unit/arena_reset.c \ $(srcroot)test/unit/arena_reset.c \
$(srcroot)test/unit/atomic.c \ $(srcroot)test/unit/atomic.c \
$(srcroot)test/unit/background_thread.c \ $(srcroot)test/unit/background_thread.c \
$(srcroot)test/unit/background_thread_enable.c \ $(srcroot)test/unit/background_thread_enable.c \
$(srcroot)test/unit/base.c \ $(srcroot)test/unit/base.c \
$(srcroot)test/unit/batch_alloc.c \
$(srcroot)test/unit/binshard.c \
$(srcroot)test/unit/bitmap.c \ $(srcroot)test/unit/bitmap.c \
$(srcroot)test/unit/bit_util.c \ $(srcroot)test/unit/bit_util.c \
$(srcroot)test/unit/binshard.c \ $(srcroot)test/unit/buf_writer.c \
$(srcroot)test/unit/cache_bin.c \
$(srcroot)test/unit/ckh.c \ $(srcroot)test/unit/ckh.c \
$(srcroot)test/unit/counter.c \
$(srcroot)test/unit/decay.c \ $(srcroot)test/unit/decay.c \
$(srcroot)test/unit/div.c \ $(srcroot)test/unit/div.c \
$(srcroot)test/unit/double_free.c \
$(srcroot)test/unit/edata_cache.c \
$(srcroot)test/unit/emitter.c \ $(srcroot)test/unit/emitter.c \
$(srcroot)test/unit/extent_quantize.c \ $(srcroot)test/unit/extent_quantize.c \
$(srcroot)test/unit/extent_util.c \ ${srcroot}test/unit/fb.c \
$(srcroot)test/unit/fork.c \ $(srcroot)test/unit/fork.c \
${srcroot}test/unit/fxp.c \
${srcroot}test/unit/san.c \
${srcroot}test/unit/san_bump.c \
$(srcroot)test/unit/hash.c \ $(srcroot)test/unit/hash.c \
$(srcroot)test/unit/hook.c \ $(srcroot)test/unit/hook.c \
$(srcroot)test/unit/hpa.c \
$(srcroot)test/unit/hpa_background_thread.c \
$(srcroot)test/unit/hpdata.c \
$(srcroot)test/unit/huge.c \ $(srcroot)test/unit/huge.c \
$(srcroot)test/unit/inspect.c \
$(srcroot)test/unit/junk.c \ $(srcroot)test/unit/junk.c \
$(srcroot)test/unit/junk_alloc.c \ $(srcroot)test/unit/junk_alloc.c \
$(srcroot)test/unit/junk_free.c \ $(srcroot)test/unit/junk_free.c \
$(srcroot)test/unit/log.c \ $(srcroot)test/unit/log.c \
$(srcroot)test/unit/mallctl.c \ $(srcroot)test/unit/mallctl.c \
$(srcroot)test/unit/malloc_conf_2.c \
$(srcroot)test/unit/malloc_io.c \ $(srcroot)test/unit/malloc_io.c \
$(srcroot)test/unit/math.c \ $(srcroot)test/unit/math.c \
$(srcroot)test/unit/mpsc_queue.c \
$(srcroot)test/unit/mq.c \ $(srcroot)test/unit/mq.c \
$(srcroot)test/unit/mtx.c \ $(srcroot)test/unit/mtx.c \
$(srcroot)test/unit/nstime.c \
$(srcroot)test/unit/oversize_threshold.c \
$(srcroot)test/unit/pa.c \
$(srcroot)test/unit/pack.c \ $(srcroot)test/unit/pack.c \
$(srcroot)test/unit/pages.c \ $(srcroot)test/unit/pages.c \
$(srcroot)test/unit/peak.c \
$(srcroot)test/unit/ph.c \ $(srcroot)test/unit/ph.c \
$(srcroot)test/unit/prng.c \ $(srcroot)test/unit/prng.c \
$(srcroot)test/unit/prof_accum.c \ $(srcroot)test/unit/prof_accum.c \
$(srcroot)test/unit/prof_active.c \ $(srcroot)test/unit/prof_active.c \
$(srcroot)test/unit/prof_gdump.c \ $(srcroot)test/unit/prof_gdump.c \
$(srcroot)test/unit/prof_hook.c \
$(srcroot)test/unit/prof_idump.c \ $(srcroot)test/unit/prof_idump.c \
$(srcroot)test/unit/prof_log.c \ $(srcroot)test/unit/prof_log.c \
$(srcroot)test/unit/prof_mdump.c \
$(srcroot)test/unit/prof_recent.c \
$(srcroot)test/unit/prof_reset.c \ $(srcroot)test/unit/prof_reset.c \
$(srcroot)test/unit/prof_stats.c \
$(srcroot)test/unit/prof_tctx.c \ $(srcroot)test/unit/prof_tctx.c \
$(srcroot)test/unit/prof_thread_name.c \ $(srcroot)test/unit/prof_thread_name.c \
$(srcroot)test/unit/prof_sys_thread_name.c \
$(srcroot)test/unit/psset.c \
$(srcroot)test/unit/ql.c \ $(srcroot)test/unit/ql.c \
$(srcroot)test/unit/qr.c \ $(srcroot)test/unit/qr.c \
$(srcroot)test/unit/rb.c \ $(srcroot)test/unit/rb.c \
$(srcroot)test/unit/retained.c \ $(srcroot)test/unit/retained.c \
$(srcroot)test/unit/rtree.c \ $(srcroot)test/unit/rtree.c \
$(srcroot)test/unit/safety_check.c \ $(srcroot)test/unit/safety_check.c \
$(srcroot)test/unit/sc.c \
$(srcroot)test/unit/sec.c \
$(srcroot)test/unit/seq.c \ $(srcroot)test/unit/seq.c \
$(srcroot)test/unit/SFMT.c \ $(srcroot)test/unit/SFMT.c \
$(srcroot)test/unit/sc.c \ $(srcroot)test/unit/size_check.c \
$(srcroot)test/unit/size_classes.c \ $(srcroot)test/unit/size_classes.c \
$(srcroot)test/unit/slab.c \ $(srcroot)test/unit/slab.c \
$(srcroot)test/unit/smoothstep.c \ $(srcroot)test/unit/smoothstep.c \
$(srcroot)test/unit/spin.c \ $(srcroot)test/unit/spin.c \
$(srcroot)test/unit/stats.c \ $(srcroot)test/unit/stats.c \
$(srcroot)test/unit/stats_print.c \ $(srcroot)test/unit/stats_print.c \
$(srcroot)test/unit/sz.c \
$(srcroot)test/unit/tcache_max.c \
$(srcroot)test/unit/test_hooks.c \ $(srcroot)test/unit/test_hooks.c \
$(srcroot)test/unit/thread_event.c \
$(srcroot)test/unit/ticker.c \ $(srcroot)test/unit/ticker.c \
$(srcroot)test/unit/nstime.c \
$(srcroot)test/unit/tsd.c \ $(srcroot)test/unit/tsd.c \
$(srcroot)test/unit/uaf.c \
$(srcroot)test/unit/witness.c \ $(srcroot)test/unit/witness.c \
$(srcroot)test/unit/zero.c $(srcroot)test/unit/zero.c \
$(srcroot)test/unit/zero_realloc_abort.c \
$(srcroot)test/unit/zero_realloc_free.c \
$(srcroot)test/unit/zero_realloc_alloc.c \
$(srcroot)test/unit/zero_reallocs.c
ifeq (@enable_prof@, 1) ifeq (@enable_prof@, 1)
TESTS_UNIT += \ TESTS_UNIT += \
$(srcroot)test/unit/arena_reset_prof.c $(srcroot)test/unit/arena_reset_prof.c \
$(srcroot)test/unit/batch_alloc_prof.c
endif endif
TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
$(srcroot)test/integration/allocated.c \ $(srcroot)test/integration/allocated.c \
...@@ -251,16 +316,26 @@ TESTS_INTEGRATION += \ ...@@ -251,16 +316,26 @@ TESTS_INTEGRATION += \
endif endif
ifeq (@enable_cxx@, 1) ifeq (@enable_cxx@, 1)
CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp
TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp \
$(srcroot)test/integration/cpp/infallible_new_true.cpp \
$(srcroot)test/integration/cpp/infallible_new_false.cpp
else else
CPP_SRCS := CPP_SRCS :=
TESTS_INTEGRATION_CPP := TESTS_INTEGRATION_CPP :=
endif endif
TESTS_STRESS := $(srcroot)test/stress/microbench.c \ TESTS_ANALYZE := $(srcroot)test/analyze/prof_bias.c \
$(srcroot)test/stress/hookbench.c $(srcroot)test/analyze/rand.c \
$(srcroot)test/analyze/sizes.c
TESTS_STRESS := $(srcroot)test/stress/batch_alloc.c \
$(srcroot)test/stress/fill_flush.c \
$(srcroot)test/stress/hookbench.c \
$(srcroot)test/stress/large_microbench.c \
$(srcroot)test/stress/mallctl.c \
$(srcroot)test/stress/microbench.c
TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_INTEGRATION_CPP) $(TESTS_STRESS) TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_INTEGRATION_CPP) \
$(TESTS_ANALYZE) $(TESTS_STRESS)
PRIVATE_NAMESPACE_HDRS := $(objroot)include/jemalloc/internal/private_namespace.h $(objroot)include/jemalloc/internal/private_namespace_jet.h PRIVATE_NAMESPACE_HDRS := $(objroot)include/jemalloc/internal/private_namespace.h $(objroot)include/jemalloc/internal/private_namespace_jet.h
PRIVATE_NAMESPACE_GEN_HDRS := $(PRIVATE_NAMESPACE_HDRS:%.h=%.gen.h) PRIVATE_NAMESPACE_GEN_HDRS := $(PRIVATE_NAMESPACE_HDRS:%.h=%.gen.h)
...@@ -276,14 +351,19 @@ C_JET_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.jet.$(O)) ...@@ -276,14 +351,19 @@ C_JET_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.jet.$(O))
C_TESTLIB_UNIT_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.unit.$(O)) C_TESTLIB_UNIT_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.unit.$(O))
C_TESTLIB_INTEGRATION_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O)) C_TESTLIB_INTEGRATION_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O))
C_UTIL_INTEGRATION_OBJS := $(C_UTIL_INTEGRATION_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O)) C_UTIL_INTEGRATION_OBJS := $(C_UTIL_INTEGRATION_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O))
C_TESTLIB_ANALYZE_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.analyze.$(O))
C_TESTLIB_STRESS_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.stress.$(O)) C_TESTLIB_STRESS_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.stress.$(O))
C_TESTLIB_OBJS := $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(C_TESTLIB_STRESS_OBJS) C_TESTLIB_OBJS := $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) \
$(C_UTIL_INTEGRATION_OBJS) $(C_TESTLIB_ANALYZE_OBJS) \
$(C_TESTLIB_STRESS_OBJS)
TESTS_UNIT_OBJS := $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%.$(O)) TESTS_UNIT_OBJS := $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%.$(O))
TESTS_INTEGRATION_OBJS := $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%.$(O)) TESTS_INTEGRATION_OBJS := $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%.$(O))
TESTS_INTEGRATION_CPP_OBJS := $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%.$(O)) TESTS_INTEGRATION_CPP_OBJS := $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%.$(O))
TESTS_ANALYZE_OBJS := $(TESTS_ANALYZE:$(srcroot)%.c=$(objroot)%.$(O))
TESTS_STRESS_OBJS := $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%.$(O)) TESTS_STRESS_OBJS := $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%.$(O))
TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS) TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_ANALYZE_OBJS) \
$(TESTS_STRESS_OBJS)
TESTS_CPP_OBJS := $(TESTS_INTEGRATION_CPP_OBJS) TESTS_CPP_OBJS := $(TESTS_INTEGRATION_CPP_OBJS)
.PHONY: all dist build_doc_html build_doc_man build_doc .PHONY: all dist build_doc_html build_doc_man build_doc
...@@ -298,7 +378,7 @@ all: build_lib ...@@ -298,7 +378,7 @@ all: build_lib
dist: build_doc dist: build_doc
$(objroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl $(objroot)doc/%$(install_suffix).html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl
ifneq ($(XSLROOT),) ifneq ($(XSLROOT),)
$(XSLTPROC) -o $@ $(objroot)doc/html.xsl $< $(XSLTPROC) -o $@ $(objroot)doc/html.xsl $<
else else
...@@ -308,9 +388,16 @@ endif ...@@ -308,9 +388,16 @@ endif
@echo "Missing xsltproc. "$@" not (re)built." @echo "Missing xsltproc. "$@" not (re)built."
endif endif
$(objroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl $(objroot)doc/%$(install_suffix).3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl
ifneq ($(XSLROOT),) ifneq ($(XSLROOT),)
$(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $< $(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $<
# The -o option (output filename) of xsltproc may not work (it uses the
# <refname> in the .xml file). Manually add the suffix if so.
ifneq ($(install_suffix),)
@if [ -f $(objroot)doc/jemalloc.3 ]; then \
mv $(objroot)doc/jemalloc.3 $(objroot)doc/jemalloc$(install_suffix).3 ; \
fi
endif
else else
ifeq ($(wildcard $(DOCS_MAN3)),) ifeq ($(wildcard $(DOCS_MAN3)),)
@echo "Missing xsltproc. Doc not built." > $@ @echo "Missing xsltproc. Doc not built." > $@
...@@ -357,12 +444,15 @@ $(C_TESTLIB_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST ...@@ -357,12 +444,15 @@ $(C_TESTLIB_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
$(C_TESTLIB_INTEGRATION_OBJS): $(objroot)test/src/%.integration.$(O): $(srcroot)test/src/%.c $(C_TESTLIB_INTEGRATION_OBJS): $(objroot)test/src/%.integration.$(O): $(srcroot)test/src/%.c
$(C_TESTLIB_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST $(C_TESTLIB_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST
$(C_UTIL_INTEGRATION_OBJS): $(objroot)src/%.integration.$(O): $(srcroot)src/%.c $(C_UTIL_INTEGRATION_OBJS): $(objroot)src/%.integration.$(O): $(srcroot)src/%.c
$(C_TESTLIB_ANALYZE_OBJS): $(objroot)test/src/%.analyze.$(O): $(srcroot)test/src/%.c
$(C_TESTLIB_ANALYZE_OBJS): CPPFLAGS += -DJEMALLOC_ANALYZE_TEST
$(C_TESTLIB_STRESS_OBJS): $(objroot)test/src/%.stress.$(O): $(srcroot)test/src/%.c $(C_TESTLIB_STRESS_OBJS): $(objroot)test/src/%.stress.$(O): $(srcroot)test/src/%.c
$(C_TESTLIB_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST -DJEMALLOC_STRESS_TESTLIB $(C_TESTLIB_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST -DJEMALLOC_STRESS_TESTLIB
$(C_TESTLIB_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include $(C_TESTLIB_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
$(TESTS_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST $(TESTS_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
$(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST $(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST
$(TESTS_INTEGRATION_CPP_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_CPP_TEST $(TESTS_INTEGRATION_CPP_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_CPP_TEST
$(TESTS_ANALYZE_OBJS): CPPFLAGS += -DJEMALLOC_ANALYZE_TEST
$(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST $(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST
$(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c $(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c
$(TESTS_CPP_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.cpp $(TESTS_CPP_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.cpp
...@@ -382,7 +472,7 @@ $(TESTS_OBJS) $(TESTS_CPP_OBJS): $(objroot)test/include/test/jemalloc_test.h ...@@ -382,7 +472,7 @@ $(TESTS_OBJS) $(TESTS_CPP_OBJS): $(objroot)test/include/test/jemalloc_test.h
endif endif
$(C_OBJS) $(CPP_OBJS) $(C_PIC_OBJS) $(CPP_PIC_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_INTEGRATION_CPP_OBJS): $(objroot)include/jemalloc/internal/private_namespace.h $(C_OBJS) $(CPP_OBJS) $(C_PIC_OBJS) $(CPP_PIC_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_INTEGRATION_CPP_OBJS): $(objroot)include/jemalloc/internal/private_namespace.h
$(C_JET_OBJS) $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_STRESS_OBJS) $(TESTS_UNIT_OBJS) $(TESTS_STRESS_OBJS): $(objroot)include/jemalloc/internal/private_namespace_jet.h $(C_JET_OBJS) $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_ANALYZE_OBJS) $(C_TESTLIB_STRESS_OBJS) $(TESTS_UNIT_OBJS) $(TESTS_ANALYZE_OBJS) $(TESTS_STRESS_OBJS): $(objroot)include/jemalloc/internal/private_namespace_jet.h
$(C_SYM_OBJS) $(C_OBJS) $(C_PIC_OBJS) $(C_JET_SYM_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O): $(C_SYM_OBJS) $(C_OBJS) $(C_PIC_OBJS) $(C_JET_SYM_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O):
@mkdir -p $(@D) @mkdir -p $(@D)
...@@ -406,7 +496,7 @@ $(objroot)include/jemalloc/internal/private_namespace_jet.gen.h: $(C_JET_SYMS) ...@@ -406,7 +496,7 @@ $(objroot)include/jemalloc/internal/private_namespace_jet.gen.h: $(C_JET_SYMS)
$(SHELL) $(srcroot)include/jemalloc/internal/private_namespace.sh $^ > $@ $(SHELL) $(srcroot)include/jemalloc/internal/private_namespace.sh $^ > $@
%.h: %.gen.h %.h: %.gen.h
@if ! `cmp -s $< $@` ; then echo "cp $< $<"; cp $< $@ ; fi @if ! `cmp -s $< $@` ; then echo "cp $< $@"; cp $< $@ ; fi
$(CPP_OBJS) $(CPP_PIC_OBJS) $(TESTS_CPP_OBJS): %.$(O): $(CPP_OBJS) $(CPP_PIC_OBJS) $(TESTS_CPP_OBJS): %.$(O):
@mkdir -p $(@D) @mkdir -p $(@D)
...@@ -445,6 +535,10 @@ $(objroot)test/integration/cpp/%$(EXE): $(objroot)test/integration/cpp/%.$(O) $( ...@@ -445,6 +535,10 @@ $(objroot)test/integration/cpp/%$(EXE): $(objroot)test/integration/cpp/%.$(O) $(
@mkdir -p $(@D) @mkdir -p $(@D)
$(CXX) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS) $(CXX) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS)
$(objroot)test/analyze/%$(EXE): $(objroot)test/analyze/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_ANALYZE_OBJS)
@mkdir -p $(@D)
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS)
$(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_STRESS_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_STRESS_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
@mkdir -p $(@D) @mkdir -p $(@D)
$(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS) $(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS)
...@@ -461,20 +555,18 @@ endif ...@@ -461,20 +555,18 @@ endif
install_bin: install_bin:
$(INSTALL) -d $(BINDIR) $(INSTALL) -d $(BINDIR)
@for b in $(BINS); do \ @for b in $(BINS); do \
echo "$(INSTALL) -m 755 $$b $(BINDIR)"; \ $(INSTALL) -v -m 755 $$b $(BINDIR); \
$(INSTALL) -m 755 $$b $(BINDIR); \
done done
install_include: install_include:
$(INSTALL) -d $(INCLUDEDIR)/jemalloc $(INSTALL) -d $(INCLUDEDIR)/jemalloc
@for h in $(C_HDRS); do \ @for h in $(C_HDRS); do \
echo "$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc"; \ $(INSTALL) -v -m 644 $$h $(INCLUDEDIR)/jemalloc; \
$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc; \
done done
install_lib_shared: $(DSOS) install_lib_shared: $(DSOS)
$(INSTALL) -d $(LIBDIR) $(INSTALL) -d $(LIBDIR)
$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR) $(INSTALL) -v -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
ifneq ($(SOREV),$(SO)) ifneq ($(SOREV),$(SO))
ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO) ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO)
endif endif
...@@ -482,15 +574,13 @@ endif ...@@ -482,15 +574,13 @@ endif
install_lib_static: $(STATIC_LIBS) install_lib_static: $(STATIC_LIBS)
$(INSTALL) -d $(LIBDIR) $(INSTALL) -d $(LIBDIR)
@for l in $(STATIC_LIBS); do \ @for l in $(STATIC_LIBS); do \
echo "$(INSTALL) -m 755 $$l $(LIBDIR)"; \ $(INSTALL) -v -m 755 $$l $(LIBDIR); \
$(INSTALL) -m 755 $$l $(LIBDIR); \
done done
install_lib_pc: $(PC) install_lib_pc: $(PC)
$(INSTALL) -d $(LIBDIR)/pkgconfig $(INSTALL) -d $(LIBDIR)/pkgconfig
@for l in $(PC); do \ @for l in $(PC); do \
echo "$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig"; \ $(INSTALL) -v -m 644 $$l $(LIBDIR)/pkgconfig; \
$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \
done done
ifeq ($(enable_shared), 1) ifeq ($(enable_shared), 1)
...@@ -501,21 +591,19 @@ install_lib: install_lib_static ...@@ -501,21 +591,19 @@ install_lib: install_lib_static
endif endif
install_lib: install_lib_pc install_lib: install_lib_pc
install_doc_html: install_doc_html: build_doc_html
$(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix) $(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix)
@for d in $(DOCS_HTML); do \ @for d in $(DOCS_HTML); do \
echo "$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \ $(INSTALL) -v -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
done done
install_doc_man: install_doc_man: build_doc_man
$(INSTALL) -d $(MANDIR)/man3 $(INSTALL) -d $(MANDIR)/man3
@for d in $(DOCS_MAN3); do \ @for d in $(DOCS_MAN3); do \
echo "$(INSTALL) -m 644 $$d $(MANDIR)/man3"; \ $(INSTALL) -v -m 644 $$d $(MANDIR)/man3; \
$(INSTALL) -m 644 $$d $(MANDIR)/man3; \
done done
install_doc: build_doc install_doc_html install_doc_man install_doc: install_doc_html install_doc_man
install: install_bin install_include install_lib install: install_bin install_include install_lib
...@@ -523,15 +611,60 @@ ifeq ($(enable_doc), 1) ...@@ -523,15 +611,60 @@ ifeq ($(enable_doc), 1)
install: install_doc install: install_doc
endif endif
uninstall_bin:
$(RM) -v $(foreach b,$(notdir $(BINS)),$(BINDIR)/$(b))
uninstall_include:
$(RM) -v $(foreach h,$(notdir $(C_HDRS)),$(INCLUDEDIR)/jemalloc/$(h))
rmdir -v $(INCLUDEDIR)/jemalloc
uninstall_lib_shared:
$(RM) -v $(LIBDIR)/$(LIBJEMALLOC).$(SOREV)
ifneq ($(SOREV),$(SO))
$(RM) -v $(LIBDIR)/$(LIBJEMALLOC).$(SO)
endif
uninstall_lib_static:
$(RM) -v $(foreach l,$(notdir $(STATIC_LIBS)),$(LIBDIR)/$(l))
uninstall_lib_pc:
$(RM) -v $(foreach p,$(notdir $(PC)),$(LIBDIR)/pkgconfig/$(p))
ifeq ($(enable_shared), 1)
uninstall_lib: uninstall_lib_shared
endif
ifeq ($(enable_static), 1)
uninstall_lib: uninstall_lib_static
endif
uninstall_lib: uninstall_lib_pc
uninstall_doc_html:
$(RM) -v $(foreach d,$(notdir $(DOCS_HTML)),$(DATADIR)/doc/jemalloc$(install_suffix)/$(d))
rmdir -v $(DATADIR)/doc/jemalloc$(install_suffix)
uninstall_doc_man:
$(RM) -v $(foreach d,$(notdir $(DOCS_MAN3)),$(MANDIR)/man3/$(d))
uninstall_doc: uninstall_doc_html uninstall_doc_man
uninstall: uninstall_bin uninstall_include uninstall_lib
ifeq ($(enable_doc), 1)
uninstall: uninstall_doc
endif
tests_unit: $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%$(EXE)) tests_unit: $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%$(EXE))
tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE)) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%$(EXE)) tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE)) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%$(EXE))
tests_analyze: $(TESTS_ANALYZE:$(srcroot)%.c=$(objroot)%$(EXE))
tests_stress: $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%$(EXE)) tests_stress: $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%$(EXE))
tests: tests_unit tests_integration tests_stress tests: tests_unit tests_integration tests_analyze tests_stress
check_unit_dir: check_unit_dir:
@mkdir -p $(objroot)test/unit @mkdir -p $(objroot)test/unit
check_integration_dir: check_integration_dir:
@mkdir -p $(objroot)test/integration @mkdir -p $(objroot)test/integration
analyze_dir:
@mkdir -p $(objroot)test/analyze
stress_dir: stress_dir:
@mkdir -p $(objroot)test/stress @mkdir -p $(objroot)test/stress
check_dir: check_unit_dir check_integration_dir check_dir: check_unit_dir check_integration_dir
...@@ -548,6 +681,12 @@ check_integration_decay: tests_integration check_integration_dir ...@@ -548,6 +681,12 @@ check_integration_decay: tests_integration check_integration_dir
$(MALLOC_CONF)="dirty_decay_ms:0,muzzy_decay_ms:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) $(MALLOC_CONF)="dirty_decay_ms:0,muzzy_decay_ms:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
check_integration: tests_integration check_integration_dir check_integration: tests_integration check_integration_dir
$(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
analyze: tests_analyze analyze_dir
ifeq ($(enable_prof), 1)
$(MALLOC_CONF)="prof:true" $(SHELL) $(objroot)test/test.sh $(TESTS_ANALYZE:$(srcroot)%.c=$(objroot)%)
else
$(SHELL) $(objroot)test/test.sh $(TESTS_ANALYZE:$(srcroot)%.c=$(objroot)%)
endif
stress: tests_stress stress_dir stress: tests_stress stress_dir
$(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%) $(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%)
check: check_unit check_integration check_integration_decay check_integration_prof check: check_unit check_integration check_integration_decay check_integration_prof
......
This document summarizes the common approaches for performance fine tuning with This document summarizes the common approaches for performance fine tuning with
jemalloc (as of 5.1.0). The default configuration of jemalloc tends to work jemalloc (as of 5.3.0). The default configuration of jemalloc tends to work
reasonably well in practice, and most applications should not have to tune any reasonably well in practice, and most applications should not have to tune any
options. However, in order to cover a wide range of applications and avoid options. However, in order to cover a wide range of applications and avoid
pathological cases, the default setting is sometimes kept conservative and pathological cases, the default setting is sometimes kept conservative and
...@@ -76,14 +76,14 @@ Examples: ...@@ -76,14 +76,14 @@ Examples:
* High resource consumption application, prioritizing memory usage: * High resource consumption application, prioritizing memory usage:
`background_thread:true` combined with shorter decay time (decreased `background_thread:true,tcache_max:4096` combined with shorter decay time
`dirty_decay_ms` and / or `muzzy_decay_ms`, (decreased `dirty_decay_ms` and / or `muzzy_decay_ms`,
e.g. `dirty_decay_ms:5000,muzzy_decay_ms:5000`), and lower arena count e.g. `dirty_decay_ms:5000,muzzy_decay_ms:5000`), and lower arena count
(e.g. number of CPUs). (e.g. number of CPUs).
* Low resource consumption application: * Low resource consumption application:
`narenas:1,lg_tcache_max:13` combined with shorter decay time (decreased `narenas:1,tcache_max:1024` combined with shorter decay time (decreased
`dirty_decay_ms` and / or `muzzy_decay_ms`,e.g. `dirty_decay_ms` and / or `muzzy_decay_ms`,e.g.
`dirty_decay_ms:1000,muzzy_decay_ms:0`). `dirty_decay_ms:1000,muzzy_decay_ms:0`).
......
...@@ -205,6 +205,8 @@ Output type: ...@@ -205,6 +205,8 @@ Output type:
--svg Generate SVG to stdout --svg Generate SVG to stdout
--gif Generate GIF to stdout --gif Generate GIF to stdout
--raw Generate symbolized jeprof data (useful with remote fetch) --raw Generate symbolized jeprof data (useful with remote fetch)
--collapsed Generate collapsed stacks for building flame graphs
(see http://www.brendangregg.com/flamegraphs.html)
Heap-Profile Options: Heap-Profile Options:
--inuse_space Display in-use (mega)bytes [default] --inuse_space Display in-use (mega)bytes [default]
...@@ -238,6 +240,7 @@ Miscellaneous: ...@@ -238,6 +240,7 @@ Miscellaneous:
--test Run unit tests --test Run unit tests
--help This message --help This message
--version Version information --version Version information
--debug-syms-by-id (Linux only) Find debug symbol files by build ID as well as by name
Environment Variables: Environment Variables:
JEPROF_TMPDIR Profiles directory. Defaults to \$HOME/jeprof JEPROF_TMPDIR Profiles directory. Defaults to \$HOME/jeprof
...@@ -332,6 +335,7 @@ sub Init() { ...@@ -332,6 +335,7 @@ sub Init() {
$main::opt_gif = 0; $main::opt_gif = 0;
$main::opt_svg = 0; $main::opt_svg = 0;
$main::opt_raw = 0; $main::opt_raw = 0;
$main::opt_collapsed = 0;
$main::opt_nodecount = 80; $main::opt_nodecount = 80;
$main::opt_nodefraction = 0.005; $main::opt_nodefraction = 0.005;
...@@ -362,6 +366,7 @@ sub Init() { ...@@ -362,6 +366,7 @@ sub Init() {
$main::opt_tools = ""; $main::opt_tools = "";
$main::opt_debug = 0; $main::opt_debug = 0;
$main::opt_test = 0; $main::opt_test = 0;
$main::opt_debug_syms_by_id = 0;
# These are undocumented flags used only by unittests. # These are undocumented flags used only by unittests.
$main::opt_test_stride = 0; $main::opt_test_stride = 0;
...@@ -405,6 +410,7 @@ sub Init() { ...@@ -405,6 +410,7 @@ sub Init() {
"svg!" => \$main::opt_svg, "svg!" => \$main::opt_svg,
"gif!" => \$main::opt_gif, "gif!" => \$main::opt_gif,
"raw!" => \$main::opt_raw, "raw!" => \$main::opt_raw,
"collapsed!" => \$main::opt_collapsed,
"interactive!" => \$main::opt_interactive, "interactive!" => \$main::opt_interactive,
"nodecount=i" => \$main::opt_nodecount, "nodecount=i" => \$main::opt_nodecount,
"nodefraction=f" => \$main::opt_nodefraction, "nodefraction=f" => \$main::opt_nodefraction,
...@@ -429,6 +435,7 @@ sub Init() { ...@@ -429,6 +435,7 @@ sub Init() {
"tools=s" => \$main::opt_tools, "tools=s" => \$main::opt_tools,
"test!" => \$main::opt_test, "test!" => \$main::opt_test,
"debug!" => \$main::opt_debug, "debug!" => \$main::opt_debug,
"debug-syms-by-id!" => \$main::opt_debug_syms_by_id,
# Undocumented flags used only by unittests: # Undocumented flags used only by unittests:
"test_stride=i" => \$main::opt_test_stride, "test_stride=i" => \$main::opt_test_stride,
) || usage("Invalid option(s)"); ) || usage("Invalid option(s)");
...@@ -490,6 +497,7 @@ sub Init() { ...@@ -490,6 +497,7 @@ sub Init() {
$main::opt_svg + $main::opt_svg +
$main::opt_gif + $main::opt_gif +
$main::opt_raw + $main::opt_raw +
$main::opt_collapsed +
$main::opt_interactive + $main::opt_interactive +
0; 0;
if ($modes > 1) { if ($modes > 1) {
...@@ -572,6 +580,11 @@ sub Init() { ...@@ -572,6 +580,11 @@ sub Init() {
foreach (@prefix_list) { foreach (@prefix_list) {
s|/+$||; s|/+$||;
} }
# Flag to prevent us from trying over and over to use
# elfutils if it's not installed (used only with
# --debug-syms-by-id option).
$main::gave_up_on_elfutils = 0;
} }
sub FilterAndPrint { sub FilterAndPrint {
...@@ -621,6 +634,8 @@ sub FilterAndPrint { ...@@ -621,6 +634,8 @@ sub FilterAndPrint {
PrintText($symbols, $flat, $cumulative, -1); PrintText($symbols, $flat, $cumulative, -1);
} elsif ($main::opt_raw) { } elsif ($main::opt_raw) {
PrintSymbolizedProfile($symbols, $profile, $main::prog); PrintSymbolizedProfile($symbols, $profile, $main::prog);
} elsif ($main::opt_collapsed) {
PrintCollapsedStacks($symbols, $profile);
} elsif ($main::opt_callgrind) { } elsif ($main::opt_callgrind) {
PrintCallgrind($calls); PrintCallgrind($calls);
} else { } else {
...@@ -2810,6 +2825,40 @@ sub IsSecondPcAlwaysTheSame { ...@@ -2810,6 +2825,40 @@ sub IsSecondPcAlwaysTheSame {
return $second_pc; return $second_pc;
} }
sub ExtractSymbolNameInlineStack {
my $symbols = shift;
my $address = shift;
my @stack = ();
if (exists $symbols->{$address}) {
my @localinlinestack = @{$symbols->{$address}};
for (my $i = $#localinlinestack; $i > 0; $i-=3) {
my $file = $localinlinestack[$i-1];
my $fn = $localinlinestack[$i-0];
if ($file eq "?" || $file eq ":0") {
$file = "??:0";
}
if ($fn eq '??') {
# If we can't get the symbol name, at least use the file information.
$fn = $file;
}
my $suffix = "[inline]";
if ($i == 2) {
$suffix = "";
}
push (@stack, $fn.$suffix);
}
}
else {
# If we can't get a symbol name, at least fill in the address.
push (@stack, $address);
}
return @stack;
}
sub ExtractSymbolLocation { sub ExtractSymbolLocation {
my $symbols = shift; my $symbols = shift;
my $address = shift; my $address = shift;
...@@ -2884,6 +2933,17 @@ sub FilterFrames { ...@@ -2884,6 +2933,17 @@ sub FilterFrames {
return $result; return $result;
} }
sub PrintCollapsedStacks {
my $symbols = shift;
my $profile = shift;
while (my ($stack_trace, $count) = each %$profile) {
my @address = split(/\n/, $stack_trace);
my @names = reverse ( map { ExtractSymbolNameInlineStack($symbols, $_) } @address );
printf("%s %d\n", join(";", @names), $count);
}
}
sub RemoveUninterestingFrames { sub RemoveUninterestingFrames {
my $symbols = shift; my $symbols = shift;
my $profile = shift; my $profile = shift;
...@@ -4440,16 +4500,54 @@ sub FindLibrary { ...@@ -4440,16 +4500,54 @@ sub FindLibrary {
# For libc libraries, the copy in /usr/lib/debug contains debugging symbols # For libc libraries, the copy in /usr/lib/debug contains debugging symbols
sub DebuggingLibrary { sub DebuggingLibrary {
my $file = shift; my $file = shift;
if ($file =~ m|^/|) {
if (-f "/usr/lib/debug$file") { if ($file !~ m|^/|) {
return "/usr/lib/debug$file"; return undef;
} elsif (-f "/usr/lib/debug$file.debug") { }
return "/usr/lib/debug$file.debug";
# Find debug symbol file if it's named after the library's name.
if (-f "/usr/lib/debug$file") {
if($main::opt_debug) { print STDERR "found debug info for $file in /usr/lib/debug$file\n"; }
return "/usr/lib/debug$file";
} elsif (-f "/usr/lib/debug$file.debug") {
if($main::opt_debug) { print STDERR "found debug info for $file in /usr/lib/debug$file.debug\n"; }
return "/usr/lib/debug$file.debug";
}
if(!$main::opt_debug_syms_by_id) {
if($main::opt_debug) { print STDERR "no debug symbols found for $file\n" };
return undef;
}
# Find debug file if it's named after the library's build ID.
my $readelf = '';
if (!$main::gave_up_on_elfutils) {
$readelf = qx/eu-readelf -n ${file}/;
if ($?) {
print STDERR "Cannot run eu-readelf. To use --debug-syms-by-id you must be on Linux, with elfutils installed.\n";
$main::gave_up_on_elfutils = 1;
return undef;
}
my $buildID = $1 if $readelf =~ /Build ID: ([A-Fa-f0-9]+)/s;
if (defined $buildID && length $buildID > 0) {
my $symbolFile = '/usr/lib/debug/.build-id/' . substr($buildID, 0, 2) . '/' . substr($buildID, 2) . '.debug';
if (-e $symbolFile) {
if($main::opt_debug) { print STDERR "found debug symbol file $symbolFile for $file\n" };
return $symbolFile;
} else {
if($main::opt_debug) { print STDERR "no debug symbol file found for $file, build ID: $buildID\n" };
return undef;
} }
}
} }
if($main::opt_debug) { print STDERR "no debug symbols found for $file, build ID unknown\n" };
return undef; return undef;
} }
# Parse text section header of a library using objdump # Parse text section header of a library using objdump
sub ParseTextSectionHeaderFromObjdump { sub ParseTextSectionHeaderFromObjdump {
my $lib = shift; my $lib = shift;
...@@ -4987,7 +5085,7 @@ sub MapToSymbols { ...@@ -4987,7 +5085,7 @@ sub MapToSymbols {
} else { } else {
# MapSymbolsWithNM tags each routine with its starting address, # MapSymbolsWithNM tags each routine with its starting address,
# useful in case the image has multiple occurrences of this # useful in case the image has multiple occurrences of this
# routine. (It uses a syntax that resembles template paramters, # routine. (It uses a syntax that resembles template parameters,
# that are automatically stripped out by ShortFunctionName().) # that are automatically stripped out by ShortFunctionName().)
# addr2line does not provide the same information. So we check # addr2line does not provide the same information. So we check
# if nm disambiguated our symbol, and if so take the annotated # if nm disambiguated our symbol, and if so take the annotated
...@@ -5339,7 +5437,7 @@ sub GetProcedureBoundaries { ...@@ -5339,7 +5437,7 @@ sub GetProcedureBoundaries {
# "nm -f $image" is supposed to fail on GNU nm, but if: # "nm -f $image" is supposed to fail on GNU nm, but if:
# #
# a. $image starts with [BbSsPp] (for example, bin/foo/bar), AND # a. $image starts with [BbSsPp] (for example, bin/foo/bar), AND
# b. you have a.out in your current directory (a not uncommon occurence) # b. you have a.out in your current directory (a not uncommon occurrence)
# #
# then "nm -f $image" succeeds because -f only looks at the first letter of # then "nm -f $image" succeeds because -f only looks at the first letter of
# the argument, which looks valid because it's [BbSsPp], and then since # the argument, which looks valid because it's [BbSsPp], and then since
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
# jemalloc profiling
This describes the mathematical basis behind jemalloc's profiling implementation, as well as the implementation tricks that make it effective. Historically, the jemalloc profiling design simply copied tcmalloc's. The implementation has since diverged, due to both the desire to record additional information, and to correct some biasing bugs.
Note: this document is markdown with embedded LaTeX; different markdown renderers may not produce the expected output. Viewing with `pandoc -s PROFILING_INTERNALS.md -o PROFILING_INTERNALS.pdf` is recommended.
## Some tricks in our implementation toolbag
### Sampling
Recording our metadata is quite expensive; we need to walk up the stack to get a stack trace. On top of that, we need to allocate storage to record that stack trace, and stick it somewhere where a profile-dumping call can find it. That call might happen on another thread, so we'll probably need to take a lock to do so. These costs are quite large compared to the average cost of an allocation. To manage this, we'll only sample some fraction of allocations. This will miss some of them, so our data will be incomplete, but we'll try to make up for it. We can tune our sampling rate to balance accuracy and performance.
### Fast Bernoulli sampling
Compared to our fast paths, even a `coinflip(p)` function can be quite expensive. Having to do a random-number generation and some floating point operations would be a sizeable relative cost. However (as pointed out in [[Vitter, 1987](https://dl.acm.org/doi/10.1145/23002.23003)]), if we can orchestrate our algorithm so that many of our `coinflip` calls share their parameter value, we can do better. We can sample from the geometric distribution, and initialize a counter with the result. When the counter hits 0, the `coinflip` function returns true (and reinitializes its internal counter).
This can let us do a random-number generation once per (logical) coinflip that comes up heads, rather than once per (logical) coinflip. Since we expect to sample relatively rarely, this can be a large win.
### Fast-path / slow-path thinking
Most programs have a skewed distribution of allocations. Smaller allocations are much more frequent than large ones, but shorter lived and less common as a fraction of program memory. "Small" and "large" are necessarily sort of fuzzy terms, but if we define "small" as "allocations jemalloc puts into slabs" and "large" as the others, then it's not uncommon for small allocations to be hundreds of times more frequent than large ones, but take up around half the amount of heap space as large ones. Moreover, small allocations tend to be much cheaper than large ones (often by a factor of 20-30): they're more likely to hit in thread caches, less likely to have to do an mmap, and cheaper to fill (by the user) once the allocation has been returned.
## An unbiased estimator of space consumption from (almost) arbitrary sampling strategies
Suppose we have a sampling strategy that meets the following criteria:
- One allocation being sampled is independent of other allocations being sampled.
- Each allocation has a non-zero probability of being sampled.
We can then estimate the bytes in live allocations through some particular stack trace as:
$$ \sum_i S_i I_i \frac{1}{\mathrm{E}[I_i]} $$
where the sum ranges over some index variable of live allocations from that stack, $S_i$ is the size of the $i$'th allocation, and $I_i$ is an indicator random variable for whether or not the $i'th$ allocation is sampled. $S_i$ and $\mathrm{E}[I_i]$ are constants (the program allocations are fixed; the random variables are the sampling decisions), so taking the expectation we get
$$ \sum_i S_i \mathrm{E}[I_i] \frac{1}{\mathrm{E}[I_i]}.$$
This is of course $\sum_i S_i$, as we want (and, a similar calculation could be done for allocation counts as well).
This is a fairly general strategy; note that while we require that sampling decisions be independent of one another's outcomes, they don't have to be independent of previous allocations, total bytes allocated, etc. You can imagine strategies that:
- Sample allocations at program startup at a higher rate than subsequent allocations
- Sample even-indexed allocations more frequently than odd-indexed ones (so long as no allocation has zero sampling probability)
- Let threads declare themselves as high-sampling-priority, and sample their allocations at an increased rate.
These can all be fit into this framework to give an unbiased estimator.
## Evaluating sampling strategies
Not all strategies for picking allocations to sample are equally good, of course. Among unbiased estimators, the lower the variance, the lower the mean squared error. Using the estimator above, the variance is:
$$
\begin{aligned}
& \mathrm{Var}[\sum_i S_i I_i \frac{1}{\mathrm{E}[I_i]}] \\
=& \sum_i \mathrm{Var}[S_i I_i \frac{1}{\mathrm{E}[I_i]}] \\
=& \sum_i \frac{S_i^2}{\mathrm{E}[I_i]^2} \mathrm{Var}[I_i] \\
=& \sum_i \frac{S_i^2}{\mathrm{E}[I_i]^2} \mathrm{Var}[I_i] \\
=& \sum_i \frac{S_i^2}{\mathrm{E}[I_i]^2} \mathrm{E}[I_i](1 - \mathrm{E}[I_i]) \\
=& \sum_i S_i^2 \frac{1 - \mathrm{E}[I_i]}{\mathrm{E}[I_i]}.
\end{aligned}
$$
We can use this formula to compare various strategy choices. All else being equal, lower-variance strategies are better.
## Possible sampling strategies
Because of the desire to avoid the fast-path costs, we'd like to use our Bernoulli trick if possible. There are two obvious counters to use: a coinflip per allocation, and a coinflip per byte allocated.
### Bernoulli sampling per-allocation
An obvious strategy is to pick some large $N$, and give each allocation a $1/N$ chance of being sampled. This would let us use our Bernoulli-via-Geometric trick. Using the formula from above, we can compute the variance as:
$$ \sum_i S_i^2 \frac{1 - \frac{1}{N}}{\frac{1}{N}} = (N-1) \sum_i S_i^2.$$
That is, an allocation of size $Z$ contributes a term of $(N-1)Z^2$ to the variance.
### Bernoulli sampling per-byte
Another option we have is to pick some rate $R$, and give each byte a $1/R$ chance of being picked for sampling (at which point we would sample its contained allocation). The chance of an allocation of size $Z$ being sampled, then, is
$$1-(1-\frac{1}{R})^{Z}$$
and an allocation of size $Z$ contributes a term of
$$Z^2 \frac{(1-\frac{1}{R})^{Z}}{1-(1-\frac{1}{R})^{Z}}.$$
In practical settings, $R$ is large, and so this is well-approximated by
$$Z^2 \frac{e^{-Z/R}}{1 - e^{-Z/R}} .$$
Just to get a sense of the dynamics here, let's look at the behavior for various values of $Z$. When $Z$ is small relative to $R$, we can use $e^z \approx 1 + x$, and conclude that the variance contributed by a small-$Z$ allocation is around
$$Z^2 \frac{1-Z/R}{Z/R} \approx RZ.$$
When $Z$ is comparable to $R$, the variance term is near $Z^2$ (we have $\frac{e^{-Z/R}}{1 - e^{-Z/R}} = 1$ when $Z/R = \ln 2 \approx 0.693$). When $Z$ is large relative to $R$, the variance term goes to zero.
## Picking a sampling strategy
The fast-path/slow-path dynamics of allocation patterns point us towards the per-byte sampling approach:
- The quadratic increase in variance per allocation in the first approach is quite costly when heaps have a non-negligible portion of their bytes in those allocations, which is practically often the case.
- The Bernoulli-per-byte approach shifts more of its samples towards large allocations, which are already a slow-path.
- We drive several tickers (e.g. tcache gc) by bytes allocated, and report bytes-allocated as a user-visible statistic, so we have to do all the necessary bookkeeping anyways.
Indeed, this is the approach we use in jemalloc. Our heap dumps record the size of the allocation and the sampling rate $R$, and jeprof unbiases by dividing by $1 - e^{-Z/R}$. The framework above would suggest dividing by $1-(1-1/R)^Z$; instead, we use the fact that $R$ is large in practical situations, and so $e^{-Z/R}$ is a good approximation (and faster to compute). (Equivalently, we may also see this as the factor that falls out from viewing sampling as a Poisson process directly).
## Consequences for heap dump consumers
Using this approach means that there are a few things users need to be aware of.
### Stack counts are not proportional to allocation frequencies
If one stack appears twice as often as another, this by itself does not imply that it allocates twice as often. Consider the case in which there are only two types of allocating call stacks in a program. Stack A allocates 8 bytes, and occurs a million times in a program. Stack B allocates 8 MB, and occurs just once in a program. If our sampling rate $R$ is about 1MB, we expect stack A to show up about 8 times, and stack B to show up once. Stack A isn't 8 times more frequent than stack B, though; it's a million times more frequent.
### Aggregation must be done after unbiasing samples
Some tools manually parse heap dump output, and aggregate across stacks (or across program runs) to provide wider-scale data analyses. When doing this aggregation, though, it's important to unbias-and-then-sum, rather than sum-and-then-unbias. Reusing our example from the previous section: suppose we collect heap dumps of the program from a million machines. We then have 8 million occurs of stack A (each of 8 bytes), and a million occurrences of stack B (each of 8 MB). If we sum first, we'll attribute 64 MB to stack A, and 8 TB to stack B. Unbiasing changes these numbers by an infinitesimal amount, so that sum-then-unbias dramatically underreports the amount of memory allocated by stack A.
## An avenue for future exploration
While the framework we laid out above is pretty general, as an engineering decision we're only interested in fairly simple approaches (i.e. ones for which the chance of an allocation being sampled depends only on its size). Our job is then: for each size class $Z$, pick a probability $p_Z$ that an allocation of that size will be sampled. We made some handwave-y references to statistical distributions to justify our choices, but there's no reason we need to pick them that way. Any set of non-zero probabilities is a valid choice.
The real limiting factor in our ability to reduce estimator variance is that fact that sampling is expensive; we want to make sure we only do it on a small fraction of allocations. Our goal, then, is to pick the $p_Z$ to minimize variance given some maximum sampling rate $P$. If we define $a_Z$ to be the fraction of allocations of size $Z$, and $l_Z$ to be the fraction of allocations of size $Z$ still alive at the time of a heap dump, then we can phrase this as an optimization problem over the choices of $p_Z$:
Minimize
$$ \sum_Z Z^2 l_Z \frac{1-p_Z}{p_Z} $$
subject to
$$ \sum_Z a_Z p_Z \leq P $$
Ignoring a term that doesn't depend on $p_Z$, the objective is minimized whenever
$$ \sum_Z Z^2 l_Z \frac{1}{p_Z} $$
is. For a particular program, $l_Z$ and $a_Z$ are just numbers that can be obtained (exactly) from existing stats introspection facilities, and we have a fairly tractable convex optimization problem (it can be framed as a second-order cone program). It would be interesting to evaluate, for various common allocation patterns, how well our current strategy adapts. Do our actual choices for $p_Z$ closely correspond to the optimal ones? How close is the variance of our choices to the variance of the optimal strategy?
You can imagine an implementation that actually goes all the way, and makes $p_Z$ selections a tuning parameter. I don't think this is a good use of development time for the foreseeable future; but I do wonder about the answers to some of these questions.
## Implementation realities
The nice story above is at least partially a lie. Initially, jeprof (copying its logic from pprof) had the sum-then-unbias error described above. The current version of jemalloc does the unbiasing step on a per-allocation basis internally, so that we're always tracking what the unbiased numbers "should" be. The problem is, actually surfacing those unbiased numbers would require a breaking change to jeprof (and the various already-deployed tools that have copied its logic). Instead, we use a little bit more trickery. Since we know at dump time the numbers we want jeprof to report, we simply choose the values we'll output so that the jeprof numbers will match the true numbers. The math is described in `src/prof_data.c` (where the only cleverness is a change of variables that lets the exponentials fall out).
This has the effect of making the output of jeprof (and related tools) correct, while making its inputs incorrect. This can be annoying to human readers of raw profiling dump output.
<svg id="Layer_3" data-name="Layer 3" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 499 184.27"><defs><style>.cls-1,.cls-3{fill:none;}.cls-2{clip-path:url(#clip-path);}.cls-3{stroke:#262262;stroke-linecap:round;stroke-linejoin:round;stroke-width:4px;}</style><clipPath id="clip-path" transform="translate(-100.66 -259.87)"><path class="cls-1" d="M144.57,396c0,18.2-9.37,27.83-37.33,23.55V400.1c11.11,2.14,12.18-.27,12.18-11.5V324.11h25Zm-12.71-78.66c-9,0-15.52-1.48-15.52-12.71S122.9,292,131.86,292s15.52,1.2,15.52,12.58C147.38,315.55,141,317.29,131.86,317.29Zm50.57,76.39c-30.64,0-35.85-18.86-35.85-35.59s5.61-35.32,35.72-35.32c35.32,0,33.44,28,33.44,40.67H170.12c.54,9.5,4,14.05,11.37,14.05,6.83,0,9.64-3.34,10-7.89l24.75.13C215.48,383.38,205.84,393.68,182.43,393.68Zm-1.47-55c-6.69,0-10,2.81-10.84,12h21.41C190.73,341.9,188.18,338.69,181,338.69Zm112.78,53.65V351.4c0-4.15-1.33-8.16-6-8.16-5,0-6,3.75-6,8.16v40.94H256.42V351.4c0-4.15-.81-8.16-5.89-8.16s-6.29,3.75-6.29,8.16v40.94H219.09V324.11h14l4.15,8c2.67-4.69,10.56-9.37,18.86-9.37,7.36,0,16.19,2.14,21,9.1,3.48-5.22,11.11-9.1,20.21-9.1,19.13,0,21.54,11.37,21.54,27.16v42.41Zm83.09,0L372.41,383c-5.48,7.22-13.11,10.7-24.35,10.7-14.85,0-26.75-6-26.75-19.93,0-15.26,12.44-20.88,44.28-23,0-9.5-1.61-12.57-8.83-12.57-6.69,0-8.56,3.48-8.56,9.9H323.45c0-12.85,6.82-25.29,32.64-25.29,30,0,34.65,14.45,34.65,31.17v38.4Zm-21.54-28.63c-6.29.94-8.3,4.28-8.3,7.36,0,4.28,2.41,6.69,8.3,6.69s10.17-4.82,10.17-15.12ZM396,392.34V297.75h24.75v94.59Zm30.77,0V297.75h24.75v94.59Zm62.21,1.34c-28.09,0-34.11-18.6-34.11-35.32s6.29-35.59,34.38-35.59c27.7,0,34.12,19,34.12,35.59C523.33,375.22,516.91,393.68,488.94,393.68Zm.27-50.84c-7.89,0-11.37,4.82-11.37,15.52s3.61,15.39,11.1,15.39c7.9,0,11.38-4.42,11.38-15.39C500.32,347.79,497.24,342.84,489.21,342.84Zm69.17,50.84c-28.9,0-34.52-18.6-34.52-35.32s5.76-35.59,34.12-35.59c21.14,0,34.52,10.84,34.52,31.17H568.42c0-9.23-5.49-11.23-10.17-11.23-7,0-11.11,4.54-11.11,15.38s4,15.52,11.11,15.52c4.81,0,10-2.41,10-10.57H592.5C592.5,383.38,579,393.68,558.38,393.68Z"/></clipPath></defs><title>jemalloc Final Logo</title><g class="cls-2"><line class="cls-3" x1="345" y1="182.27" x2="345" y2="2"/><line class="cls-3" x1="225" y1="182.27" x2="225" y2="2"/><line class="cls-3" x1="105" y1="182.27" x2="105" y2="2"/><line class="cls-3" x1="43" y1="182.27" x2="43" y2="2"/><line class="cls-3" x1="475" y1="182.27" x2="475" y2="2"/><line class="cls-3" x1="195" y1="182.27" x2="195" y2="2"/><line class="cls-3" x1="75" y1="182.27" x2="75" y2="2"/><line class="cls-3" x1="337" y1="182.27" x2="337" y2="2"/><line class="cls-3" x1="215" y1="182.27" x2="215" y2="2"/><line class="cls-3" x1="95" y1="182.27" x2="95" y2="2"/><line class="cls-3" x1="415" y1="182.27" x2="415" y2="2"/><line class="cls-3" x1="385" y1="182.27" x2="385" y2="2"/><line class="cls-3" x1="183" y1="182.27" x2="183" y2="2"/><line class="cls-3" x1="65" y1="182.27" x2="65" y2="2"/><line class="cls-3" x1="173" y1="182.27" x2="173" y2="2"/><line class="cls-3" x1="145" y1="182.27" x2="145" y2="2"/><line class="cls-3" x1="163" y1="182.27" x2="163" y2="2"/><line class="cls-3" x1="460" y1="182.27" x2="460" y2="2"/><line class="cls-3" x1="281" y1="182.27" x2="281" y2="2"/><line class="cls-3" x1="313" y1="182.27" x2="313" y2="2"/><line class="cls-3" x1="252" y1="182.27" x2="252" y2="2"/><line class="cls-3" x1="450" y1="182.27" x2="450" y2="2"/><line class="cls-3" x1="271" y1="182.27" x2="271" y2="2"/><line class="cls-3" x1="332" y1="182.27" x2="332" y2="2"/><line class="cls-3" x1="203" y1="182.27" x2="203" y2="2"/><line class="cls-3" x1="13" y1="182.27" x2="13" y2="2"/><line class="cls-3" x1="373" y1="182.27" x2="373" y2="2"/><line class="cls-3" x1="354" y1="182.27" x2="354" y2="2"/><line class="cls-3" x1="235" y1="182.27" x2="235" y2="2"/><line class="cls-3" x1="115" y1="182.27" x2="115" y2="2"/><line class="cls-3" x1="53" y1="182.27" x2="53" y2="2"/><line class="cls-3" x1="484" y1="182.27" x2="484" y2="2"/><line class="cls-3" x1="405" y1="182.27" x2="405" y2="2"/><line class="cls-3" x1="85" y1="182.27" x2="85" y2="2"/><line class="cls-3" x1="225" y1="182.27" x2="225" y2="2"/><line class="cls-3" x1="105" y1="182.27" x2="105" y2="2"/><line class="cls-3" x1="43" y1="182.27" x2="43" y2="2"/><line class="cls-3" x1="435" y1="182.27" x2="435" y2="2"/><line class="cls-3" x1="123" y1="182.27" x2="123" y2="2"/><line class="cls-3" x1="75" y1="182.27" x2="75" y2="2"/><line class="cls-3" x1="183" y1="182.27" x2="183" y2="2"/><line class="cls-3" x1="155" y1="182.27" x2="155" y2="2"/><line class="cls-3" x1="173" y1="182.27" x2="173" y2="2"/><line class="cls-3" x1="145" y1="182.27" x2="145" y2="2"/><line class="cls-3" x1="470" y1="182.27" x2="470" y2="2"/><line class="cls-3" x1="292" y1="182.27" x2="292" y2="2"/><line class="cls-3" x1="262" y1="182.27" x2="262" y2="2"/><line class="cls-3" x1="460" y1="182.27" x2="460" y2="2"/><line class="cls-3" x1="281" y1="182.27" x2="281" y2="2"/><line class="cls-3" x1="313" y1="182.27" x2="313" y2="2"/><line class="cls-3" x1="243" y1="182.27" x2="243" y2="2"/><line class="cls-3" x1="22" y1="182.27" x2="22" y2="2"/><line class="cls-3" x1="383" y1="182.27" x2="383" y2="2"/><line class="cls-3" x1="5" y1="182.27" x2="5" y2="2"/><line class="cls-3" x1="133" y1="182.27" x2="133" y2="2"/><line class="cls-3" x1="362" y1="182.27" x2="362" y2="2"/><line class="cls-3" x1="288" y1="182.27" x2="288" y2="2"/><line class="cls-3" x1="298" y1="182.27" x2="298" y2="2"/><line class="cls-3" x1="423" y1="182.27" x2="423" y2="2"/><line class="cls-3" x1="369" y1="182.27" x2="369" y2="2"/><line class="cls-3" x1="490" y1="182.27" x2="490" y2="2"/><line class="cls-3" x1="2" y1="182.27" x2="2" y2="2"/><line class="cls-3" x1="493" y1="182.27" x2="493" y2="2"/><line class="cls-3" x1="225" y1="182.27" x2="225" y2="2"/><line class="cls-3" x1="105" y1="182.27" x2="105" y2="2"/><line class="cls-3" x1="43" y1="182.27" x2="43" y2="2"/><line class="cls-3" x1="475" y1="182.27" x2="475" y2="2"/><line class="cls-3" x1="195" y1="182.27" x2="195" y2="2"/><line class="cls-3" x1="75" y1="182.27" x2="75" y2="2"/><line class="cls-3" x1="337" y1="182.27" x2="337" y2="2"/><line class="cls-3" x1="215" y1="182.27" x2="215" y2="2"/><line class="cls-3" x1="95" y1="182.27" x2="95" y2="2"/><line class="cls-3" x1="415" y1="182.27" x2="415" y2="2"/><line class="cls-3" x1="385" y1="182.27" x2="385" y2="2"/><line class="cls-3" x1="183" y1="182.27" x2="183" y2="2"/><line class="cls-3" x1="65" y1="182.27" x2="65" y2="2"/><line class="cls-3" x1="173" y1="182.27" x2="173" y2="2"/><line class="cls-3" x1="145" y1="182.27" x2="145" y2="2"/><line class="cls-3" x1="163" y1="182.27" x2="163" y2="2"/><line class="cls-3" x1="460" y1="182.27" x2="460" y2="2"/><line class="cls-3" x1="281" y1="182.27" x2="281" y2="2"/><line class="cls-3" x1="313" y1="182.27" x2="313" y2="2"/><line class="cls-3" x1="252" y1="182.27" x2="252" y2="2"/><line class="cls-3" x1="450" y1="182.27" x2="450" y2="2"/><line class="cls-3" x1="271" y1="182.27" x2="271" y2="2"/><line class="cls-3" x1="306" y1="182.27" x2="306" y2="2"/><line class="cls-3" x1="203" y1="182.27" x2="203" y2="2"/><line class="cls-3" x1="13" y1="182.27" x2="13" y2="2"/><line class="cls-3" x1="373" y1="182.27" x2="373" y2="2"/><line class="cls-3" x1="354" y1="182.27" x2="354" y2="2"/><line class="cls-3" x1="235" y1="182.27" x2="235" y2="2"/><line class="cls-3" x1="115" y1="182.27" x2="115" y2="2"/><line class="cls-3" x1="53" y1="182.27" x2="53" y2="2"/><line class="cls-3" x1="484" y1="182.27" x2="484" y2="2"/><line class="cls-3" x1="405" y1="182.27" x2="405" y2="2"/><line class="cls-3" x1="85" y1="182.27" x2="85" y2="2"/><line class="cls-3" x1="225" y1="182.27" x2="225" y2="2"/><line class="cls-3" x1="105" y1="182.27" x2="105" y2="2"/><line class="cls-3" x1="43" y1="182.27" x2="43" y2="2"/><line class="cls-3" x1="435" y1="182.27" x2="435" y2="2"/><line class="cls-3" x1="123" y1="182.27" x2="123" y2="2"/><line class="cls-3" x1="75" y1="182.27" x2="75" y2="2"/><line class="cls-3" x1="183" y1="182.27" x2="183" y2="2"/><line class="cls-3" x1="155" y1="182.27" x2="155" y2="2"/><line class="cls-3" x1="173" y1="182.27" x2="173" y2="2"/><line class="cls-3" x1="145" y1="182.27" x2="145" y2="2"/><line class="cls-3" x1="470" y1="182.27" x2="470" y2="2"/><line class="cls-3" x1="292" y1="182.27" x2="292" y2="2"/><line class="cls-3" x1="262" y1="182.27" x2="262" y2="2"/><line class="cls-3" x1="460" y1="182.27" x2="460" y2="2"/><line class="cls-3" x1="281" y1="182.27" x2="281" y2="2"/><line class="cls-3" x1="328" y1="182.27" x2="328" y2="2"/><line class="cls-3" x1="243" y1="182.27" x2="243" y2="2"/><line class="cls-3" x1="22" y1="182.27" x2="22" y2="2"/><line class="cls-3" x1="383" y1="182.27" x2="383" y2="2"/><line class="cls-3" x1="5" y1="182.27" x2="5" y2="2"/><line class="cls-3" x1="32" y1="182.27" x2="32" y2="2"/><line class="cls-3" x1="133" y1="182.27" x2="133" y2="2"/><line class="cls-3" x1="362" y1="182.27" x2="362" y2="2"/><line class="cls-3" x1="288" y1="182.27" x2="288" y2="2"/><line class="cls-3" x1="298" y1="182.27" x2="298" y2="2"/><line class="cls-3" x1="423" y1="182.27" x2="423" y2="2"/><line class="cls-3" x1="369" y1="182.27" x2="369" y2="2"/><line class="cls-3" x1="490" y1="182.27" x2="490" y2="2"/><line class="cls-3" x1="2" y1="182.27" x2="2" y2="2"/><line class="cls-3" x1="493" y1="182.27" x2="493" y2="2"/><line class="cls-3" x1="349" y1="182.27" x2="349" y2="2"/><line class="cls-3" x1="229" y1="182.27" x2="229" y2="2"/><line class="cls-3" x1="109" y1="182.27" x2="109" y2="2"/><line class="cls-3" x1="47" y1="182.27" x2="47" y2="2"/><line class="cls-3" x1="479" y1="182.27" x2="479" y2="2"/><line class="cls-3" x1="399" y1="182.27" x2="399" y2="2"/><line class="cls-3" x1="199" y1="182.27" x2="199" y2="2"/><line class="cls-3" x1="79" y1="182.27" x2="79" y2="2"/><line class="cls-3" x1="341" y1="182.27" x2="341" y2="2"/><line class="cls-3" x1="219" y1="182.27" x2="219" y2="2"/><line class="cls-3" x1="99" y1="182.27" x2="99" y2="2"/><line class="cls-3" x1="41" y1="182.27" x2="41" y2="2"/><line class="cls-3" x1="419" y1="182.27" x2="419" y2="2"/><line class="cls-3" x1="389" y1="182.27" x2="389" y2="2"/><line class="cls-3" x1="187" y1="182.27" x2="187" y2="2"/><line class="cls-3" x1="69" y1="182.27" x2="69" y2="2"/><line class="cls-3" x1="177" y1="182.27" x2="177" y2="2"/><line class="cls-3" x1="149" y1="182.27" x2="149" y2="2"/><line class="cls-3" x1="464" y1="182.27" x2="464" y2="2"/><line class="cls-3" x1="285" y1="182.27" x2="285" y2="2"/><line class="cls-3" x1="317" y1="182.27" x2="317" y2="2"/><line class="cls-3" x1="454" y1="182.27" x2="454" y2="2"/><line class="cls-3" x1="275" y1="182.27" x2="275" y2="2"/><line class="cls-3" x1="308" y1="182.27" x2="308" y2="2"/><line class="cls-3" x1="207" y1="182.27" x2="207" y2="2"/><line class="cls-3" x1="17" y1="182.27" x2="17" y2="2"/><line class="cls-3" x1="377" y1="182.27" x2="377" y2="2"/><line class="cls-3" x1="358" y1="182.27" x2="358" y2="2"/><line class="cls-3" x1="238" y1="182.27" x2="238" y2="2"/><line class="cls-3" x1="119" y1="182.27" x2="119" y2="2"/><line class="cls-3" x1="488" y1="182.27" x2="488" y2="2"/><line class="cls-3" x1="409" y1="182.27" x2="409" y2="2"/><line class="cls-3" x1="229" y1="182.27" x2="229" y2="2"/><line class="cls-3" x1="109" y1="182.27" x2="109" y2="2"/><line class="cls-3" x1="47" y1="182.27" x2="47" y2="2"/><line class="cls-3" x1="439" y1="182.27" x2="439" y2="2"/><line class="cls-3" x1="399" y1="182.27" x2="399" y2="2"/><line class="cls-3" x1="127" y1="182.27" x2="127" y2="2"/><line class="cls-3" x1="79" y1="182.27" x2="79" y2="2"/><line class="cls-3" x1="187" y1="182.27" x2="187" y2="2"/><line class="cls-3" x1="159" y1="182.27" x2="159" y2="2"/><line class="cls-3" x1="177" y1="182.27" x2="177" y2="2"/><line class="cls-3" x1="149" y1="182.27" x2="149" y2="2"/><line class="cls-3" x1="474" y1="182.27" x2="474" y2="2"/><line class="cls-3" x1="266" y1="182.27" x2="266" y2="2"/><line class="cls-3" x1="464" y1="182.27" x2="464" y2="2"/><line class="cls-3" x1="285" y1="182.27" x2="285" y2="2"/><line class="cls-3" x1="317" y1="182.27" x2="317" y2="2"/><line class="cls-3" x1="247" y1="182.27" x2="247" y2="2"/><line class="cls-3" x1="26" y1="182.27" x2="26" y2="2"/><line class="cls-3" x1="387" y1="182.27" x2="387" y2="2"/><line class="cls-3" x1="9" y1="182.27" x2="9" y2="2"/><line class="cls-3" x1="137" y1="182.27" x2="137" y2="2"/><line class="cls-3" x1="292" y1="182.27" x2="292" y2="2"/><line class="cls-3" x1="373" y1="182.27" x2="373" y2="2"/><line class="cls-3" x1="56" y1="182.27" x2="56" y2="2"/><line class="cls-3" x1="494" y1="182.27" x2="494" y2="2"/><line class="cls-3" x1="497" y1="182.27" x2="497" y2="2"/><line class="cls-3" x1="349" y1="182.27" x2="349" y2="2"/><line class="cls-3" x1="229" y1="182.27" x2="229" y2="2"/><line class="cls-3" x1="109" y1="182.27" x2="109" y2="2"/><line class="cls-3" x1="47" y1="182.27" x2="47" y2="2"/><line class="cls-3" x1="479" y1="182.27" x2="479" y2="2"/><line class="cls-3" x1="399" y1="182.27" x2="399" y2="2"/><line class="cls-3" x1="199" y1="182.27" x2="199" y2="2"/><line class="cls-3" x1="79" y1="182.27" x2="79" y2="2"/><line class="cls-3" x1="341" y1="182.27" x2="341" y2="2"/><line class="cls-3" x1="219" y1="182.27" x2="219" y2="2"/><line class="cls-3" x1="99" y1="182.27" x2="99" y2="2"/><line class="cls-3" x1="41" y1="182.27" x2="41" y2="2"/><line class="cls-3" x1="419" y1="182.27" x2="419" y2="2"/><line class="cls-3" x1="389" y1="182.27" x2="389" y2="2"/><line class="cls-3" x1="187" y1="182.27" x2="187" y2="2"/><line class="cls-3" x1="69" y1="182.27" x2="69" y2="2"/><line class="cls-3" x1="177" y1="182.27" x2="177" y2="2"/><line class="cls-3" x1="149" y1="182.27" x2="149" y2="2"/><line class="cls-3" x1="141" y1="182.27" x2="141" y2="2"/><line class="cls-3" x1="464" y1="182.27" x2="464" y2="2"/><line class="cls-3" x1="285" y1="182.27" x2="285" y2="2"/><line class="cls-3" x1="317" y1="182.27" x2="317" y2="2"/><line class="cls-3" x1="454" y1="182.27" x2="454" y2="2"/><line class="cls-3" x1="275" y1="182.27" x2="275" y2="2"/><line class="cls-3" x1="308" y1="182.27" x2="308" y2="2"/><line class="cls-3" x1="207" y1="182.27" x2="207" y2="2"/><line class="cls-3" x1="17" y1="182.27" x2="17" y2="2"/><line class="cls-3" x1="377" y1="182.27" x2="377" y2="2"/><line class="cls-3" x1="119" y1="182.27" x2="119" y2="2"/><line class="cls-3" x1="488" y1="182.27" x2="488" y2="2"/><line class="cls-3" x1="409" y1="182.27" x2="409" y2="2"/><line class="cls-3" x1="229" y1="182.27" x2="229" y2="2"/><line class="cls-3" x1="109" y1="182.27" x2="109" y2="2"/><line class="cls-3" x1="47" y1="182.27" x2="47" y2="2"/><line class="cls-3" x1="439" y1="182.27" x2="439" y2="2"/><line class="cls-3" x1="399" y1="182.27" x2="399" y2="2"/><line class="cls-3" x1="127" y1="182.27" x2="127" y2="2"/><line class="cls-3" x1="79" y1="182.27" x2="79" y2="2"/><line class="cls-3" x1="187" y1="182.27" x2="187" y2="2"/><line class="cls-3" x1="159" y1="182.27" x2="159" y2="2"/><line class="cls-3" x1="177" y1="182.27" x2="177" y2="2"/><line class="cls-3" x1="149" y1="182.27" x2="149" y2="2"/><line class="cls-3" x1="474" y1="182.27" x2="474" y2="2"/><line class="cls-3" x1="295" y1="182.27" x2="295" y2="2"/><line class="cls-3" x1="266" y1="182.27" x2="266" y2="2"/><line class="cls-3" x1="464" y1="182.27" x2="464" y2="2"/><line class="cls-3" x1="285" y1="182.27" x2="285" y2="2"/><line class="cls-3" x1="317" y1="182.27" x2="317" y2="2"/><line class="cls-3" x1="247" y1="182.27" x2="247" y2="2"/><line class="cls-3" x1="58" y1="182.27" x2="58" y2="2"/><line class="cls-3" x1="387" y1="182.27" x2="387" y2="2"/><line class="cls-3" x1="9" y1="182.27" x2="9" y2="2"/><line class="cls-3" x1="292" y1="182.27" x2="292" y2="2"/><line class="cls-3" x1="301" y1="182.27" x2="301" y2="2"/><line class="cls-3" x1="428" y1="182.27" x2="428" y2="2"/><line class="cls-3" x1="373" y1="182.27" x2="373" y2="2"/><line class="cls-3" x1="56" y1="182.27" x2="56" y2="2"/><line class="cls-3" x1="494" y1="182.27" x2="494" y2="2"/><line class="cls-3" x1="497" y1="182.27" x2="497" y2="2"/></g></svg>
\ No newline at end of file
#ifndef JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H
#define JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H
/*
* The callback to be executed "periodically", in response to some amount of
* allocator activity.
*
* This callback need not be computing any sort of peak (although that's the
* intended first use case), but we drive it from the peak counter, so it's
* keeps things tidy to keep it here.
*
* The calls to this thunk get driven by the peak_event module.
*/
#define ACTIVITY_CALLBACK_THUNK_INITIALIZER {NULL, NULL}
typedef void (*activity_callback_t)(void *uctx, uint64_t allocated,
uint64_t deallocated);
typedef struct activity_callback_thunk_s activity_callback_thunk_t;
struct activity_callback_thunk_s {
activity_callback_t callback;
void *uctx;
};
#endif /* JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H */
...@@ -2,59 +2,67 @@ ...@@ -2,59 +2,67 @@
#define JEMALLOC_INTERNAL_ARENA_EXTERNS_H #define JEMALLOC_INTERNAL_ARENA_EXTERNS_H
#include "jemalloc/internal/bin.h" #include "jemalloc/internal/bin.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/hook.h" #include "jemalloc/internal/hook.h"
#include "jemalloc/internal/pages.h" #include "jemalloc/internal/pages.h"
#include "jemalloc/internal/stats.h" #include "jemalloc/internal/stats.h"
/*
* When the amount of pages to be purged exceeds this amount, deferred purge
* should happen.
*/
#define ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
extern ssize_t opt_dirty_decay_ms; extern ssize_t opt_dirty_decay_ms;
extern ssize_t opt_muzzy_decay_ms; extern ssize_t opt_muzzy_decay_ms;
extern percpu_arena_mode_t opt_percpu_arena; extern percpu_arena_mode_t opt_percpu_arena;
extern const char *percpu_arena_mode_names[]; extern const char *percpu_arena_mode_names[];
extern const uint64_t h_steps[SMOOTHSTEP_NSTEPS]; extern div_info_t arena_binind_div_info[SC_NBINS];
extern malloc_mutex_t arenas_lock; extern malloc_mutex_t arenas_lock;
extern emap_t arena_emap_global;
extern size_t opt_oversize_threshold; extern size_t opt_oversize_threshold;
extern size_t oversize_threshold; extern size_t oversize_threshold;
/*
* arena_bin_offsets[binind] is the offset of the first bin shard for size class
* binind.
*/
extern uint32_t arena_bin_offsets[SC_NBINS];
void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena,
unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms,
ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy); ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
bin_stats_t *bstats, arena_stats_large_t *lstats, bin_stats_data_t *bstats, arena_stats_large_t *lstats,
arena_stats_extents_t *estats); pac_estats_t *estats, hpa_shard_stats_t *hpastats, sec_stats_t *secstats);
void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
extent_hooks_t **r_extent_hooks, extent_t *extent); edata_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena,
#ifdef JEMALLOC_JET size_t usize, size_t alignment, bool zero);
size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr);
#endif
extent_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena,
size_t usize, size_t alignment, bool *zero);
void arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, void arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena,
extent_t *extent); edata_t *edata);
void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t oldsize); edata_t *edata, size_t oldsize);
void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t oldsize); edata_t *edata, size_t oldsize);
ssize_t arena_dirty_decay_ms_get(arena_t *arena); bool arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, extent_state_t state,
bool arena_dirty_decay_ms_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_ms); ssize_t decay_ms);
ssize_t arena_muzzy_decay_ms_get(arena_t *arena); ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
bool arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_ms);
void arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, void arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
bool all); bool all);
uint64_t arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
void arena_reset(tsd_t *tsd, arena_t *arena); void arena_reset(tsd_t *tsd, arena_t *arena);
void arena_destroy(tsd_t *tsd, arena_t *arena); void arena_destroy(tsd_t *tsd, arena_t *arena);
void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, void arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena,
cache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes); cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind,
void arena_alloc_junk_small(void *ptr, const bin_info_t *bin_info, const unsigned nfill);
bool zero);
typedef void (arena_dalloc_junk_small_t)(void *, const bin_info_t *);
extern arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small;
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
szind_t ind, bool zero); szind_t ind, bool zero);
...@@ -63,8 +71,12 @@ void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, ...@@ -63,8 +71,12 @@ void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
void arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize); void arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize);
void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
bool slow_path); bool slow_path);
void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
szind_t binind, extent_t *extent, void *ptr);
void arena_dalloc_bin_locked_handle_newly_empty(tsdn_t *tsdn, arena_t *arena,
edata_t *slab, bin_t *bin);
void arena_dalloc_bin_locked_handle_newly_nonempty(tsdn_t *tsdn, arena_t *arena,
edata_t *slab, bin_t *bin);
void arena_dalloc_small(tsdn_t *tsdn, void *ptr); void arena_dalloc_small(tsdn_t *tsdn, void *ptr);
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero, size_t *newsize); size_t extra, bool zero, size_t *newsize);
...@@ -72,6 +84,9 @@ void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, ...@@ -72,6 +84,9 @@ void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, tcache_t *tcache, size_t size, size_t alignment, bool zero, tcache_t *tcache,
hook_ralloc_args_t *hook_args); hook_ralloc_args_t *hook_args);
dss_prec_t arena_dss_prec_get(arena_t *arena); dss_prec_t arena_dss_prec_get(arena_t *arena);
ehooks_t *arena_get_ehooks(arena_t *arena);
extent_hooks_t *arena_set_extent_hooks(tsd_t *tsd, arena_t *arena,
extent_hooks_t *extent_hooks);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
ssize_t arena_dirty_decay_ms_default_get(void); ssize_t arena_dirty_decay_ms_default_get(void);
bool arena_dirty_decay_ms_default_set(ssize_t decay_ms); bool arena_dirty_decay_ms_default_set(ssize_t decay_ms);
...@@ -82,14 +97,15 @@ bool arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, ...@@ -82,14 +97,15 @@ bool arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena,
unsigned arena_nthreads_get(arena_t *arena, bool internal); unsigned arena_nthreads_get(arena_t *arena, bool internal);
void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal);
size_t arena_extent_sn_next(arena_t *arena); arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
arena_t *arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
bool arena_init_huge(void); bool arena_init_huge(void);
bool arena_is_huge(unsigned arena_ind); bool arena_is_huge(unsigned arena_ind);
arena_t *arena_choose_huge(tsd_t *tsd); arena_t *arena_choose_huge(tsd_t *tsd);
bin_t *arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, bin_t *arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
unsigned *binshard); unsigned *binshard);
void arena_boot(sc_data_t *sc_data); size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
void **ptrs, size_t nfill, bool zero);
bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
void arena_prefork0(tsdn_t *tsdn, arena_t *arena); void arena_prefork0(tsdn_t *tsdn, arena_t *arena);
void arena_prefork1(tsdn_t *tsdn, arena_t *arena); void arena_prefork1(tsdn_t *tsdn, arena_t *arena);
void arena_prefork2(tsdn_t *tsdn, arena_t *arena); void arena_prefork2(tsdn_t *tsdn, arena_t *arena);
...@@ -98,6 +114,7 @@ void arena_prefork4(tsdn_t *tsdn, arena_t *arena); ...@@ -98,6 +114,7 @@ void arena_prefork4(tsdn_t *tsdn, arena_t *arena);
void arena_prefork5(tsdn_t *tsdn, arena_t *arena); void arena_prefork5(tsdn_t *tsdn, arena_t *arena);
void arena_prefork6(tsdn_t *tsdn, arena_t *arena); void arena_prefork6(tsdn_t *tsdn, arena_t *arena);
void arena_prefork7(tsdn_t *tsdn, arena_t *arena); void arena_prefork7(tsdn_t *tsdn, arena_t *arena);
void arena_prefork8(tsdn_t *tsdn, arena_t *arena);
void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena); void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
void arena_postfork_child(tsdn_t *tsdn, arena_t *arena); void arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
static inline unsigned static inline unsigned
arena_ind_get(const arena_t *arena) { arena_ind_get(const arena_t *arena) {
return base_ind_get(arena->base); return arena->ind;
} }
static inline void static inline void
...@@ -21,37 +21,4 @@ arena_internal_get(arena_t *arena) { ...@@ -21,37 +21,4 @@ arena_internal_get(arena_t *arena) {
return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED); return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
} }
static inline bool
arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes) {
cassert(config_prof);
if (likely(prof_interval == 0 || !prof_active_get_unlocked())) {
return false;
}
return prof_accum_add(tsdn, &arena->prof_accum, accumbytes);
}
static inline void
percpu_arena_update(tsd_t *tsd, unsigned cpu) {
assert(have_percpu_arena);
arena_t *oldarena = tsd_arena_get(tsd);
assert(oldarena != NULL);
unsigned oldind = arena_ind_get(oldarena);
if (oldind != cpu) {
unsigned newind = cpu;
arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
assert(newarena != NULL);
/* Set new arena/tcache associations. */
arena_migrate(tsd, oldind, newind);
tcache_t *tcache = tcache_get(tsd);
if (tcache != NULL) {
tcache_arena_reassociate(tsd_tsdn(tsd), tcache,
newarena);
}
}
}
#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_A_H */ #endif /* JEMALLOC_INTERNAL_ARENA_INLINES_A_H */
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment