Unverified Commit 9ebf80a2 authored by Ozan Tezcan's avatar Ozan Tezcan Committed by GitHub
Browse files

Fix memory leak of jemalloc tcache on function flush command (#13661)

Starting from https://github.com/redis/redis/pull/13133

, we allocate a
jemalloc thread cache and use it for lua vm.
On certain cases, like `script flush` or `function flush` command, we
free the existing thread cache and create a new one.

Though, for `function flush`, we were not actually destroying the
existing thread cache itself. Each call creates a new thread cache on
jemalloc and we leak the previous thread cache instances. Jemalloc
allows maximum 4096 thread cache instances. If we reach this limit,
Redis prints "Failed creating the lua jemalloc tcache" log and abort.

There are other cases that can cause this memory leak, including
replication scenarios when emptyData() is called.

The implication is that it looks like redis `used_memory` is low, but
`allocator_allocated` and RSS remain high.
Co-authored-by: default avatardebing.sun <debing.sun@redis.com>
parent 15563450
......@@ -259,12 +259,16 @@ void scriptingInit(int setup) {
void freeLuaScriptsSync(dict *lua_scripts, list *lua_scripts_lru_list, lua_State *lua) {
dictRelease(lua_scripts);
listRelease(lua_scripts_lru_list);
lua_close(lua);
#if defined(USE_JEMALLOC)
/* When lua is closed, destroy the previously used private tcache. */
void *ud = (global_State*)G(lua)->ud;
unsigned int lua_tcache = (unsigned int)(uintptr_t)ud;
#endif
lua_close(lua);
#if defined(USE_JEMALLOC)
je_mallctl("tcache.destroy", NULL, NULL, (void *)&lua_tcache, sizeof(unsigned int));
#endif
}
......
......@@ -23,6 +23,9 @@
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
#if defined(USE_JEMALLOC)
#include <lstate.h>
#endif
#define LUA_ENGINE_NAME "LUA"
#define REGISTRY_ENGINE_CTX_NAME "__ENGINE_CTX__"
......@@ -189,8 +192,18 @@ static void luaEngineFreeFunction(void *engine_ctx, void *compiled_function) {
static void luaEngineFreeCtx(void *engine_ctx) {
luaEngineCtx *lua_engine_ctx = engine_ctx;
#if defined(USE_JEMALLOC)
/* When lua is closed, destroy the previously used private tcache. */
void *ud = (global_State*)G(lua_engine_ctx->lua)->ud;
unsigned int lua_tcache = (unsigned int)(uintptr_t)ud;
#endif
lua_close(lua_engine_ctx->lua);
zfree(lua_engine_ctx);
#if defined(USE_JEMALLOC)
je_mallctl("tcache.destroy", NULL, NULL, (void *)&lua_tcache, sizeof(unsigned int));
#endif
}
static void luaRegisterFunctionArgsInitialize(registerFunctionArgs *register_f_args,
......
......@@ -64,7 +64,7 @@ lua_State *createLuaState(void) {
size_t sz = sizeof(unsigned int);
int err = je_mallctl("tcache.create", (void *)&tcache, &sz, NULL, 0);
if (err) {
serverLog(LL_WARNING, "Failed creating the lua jemalloc tcache.");
serverLog(LL_WARNING, "Failed creating the lua jemalloc tcache (err=%d).", err);
exit(1);
}
......@@ -79,7 +79,7 @@ void luaEnvInit(void) {
size_t sz = sizeof(unsigned int);
int err = je_mallctl("arenas.create", (void *)&arena, &sz, NULL, 0);
if (err) {
serverLog(LL_WARNING, "Failed creating the lua jemalloc arena.");
serverLog(LL_WARNING, "Failed creating the lua jemalloc arena (err=%d).", err);
exit(1);
}
server.lua_arena = arena;
......
......@@ -1884,6 +1884,35 @@ start_server {tags {"scripting needs:debug"}} {
}
start_server {tags {"scripting"}} {
test "Test script flush will not leak memory - script:$is_eval" {
r flushall
r script flush
r function flush
# This is a best-effort test to check we don't leak some resources on
# script flush and function flush commands. For lua vm, we create a
# jemalloc thread cache. On each script flush command, thread cache is
# destroyed and we create a new one. In this test, running script flush
# many times to verify there is no increase in the memory usage while
# re-creating some of the resources for lua vm.
set used_memory [s used_memory]
set allocator_allocated [s allocator_allocated]
r multi
for {set j 1} {$j <= 500} {incr j} {
if {$is_eval} {
r SCRIPT FLUSH
} else {
r FUNCTION FLUSH
}
}
r exec
# Verify used memory is not (much) higher.
assert_lessthan [s used_memory] [expr $used_memory*1.5]
assert_lessthan [s allocator_allocated] [expr $allocator_allocated*1.5]
}
test "Verify Lua performs GC correctly after script loading" {
set dummy_script "--[string repeat x 10]\nreturn "
set n 50000
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment