Unverified Commit 79fd2558 authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Add Lua VM memory to memory overhead, now that it's part of zmalloc (#13660)

To complement the work done in #13133.
it added the script VMs memory to be counted as part of zmalloc, but
that means they
should be also counted as part of the non-value overhead.

this commit contains some refactoring to make variable names and
function names less confusing.
it also adds a new field named `script.VMs` into the `MEMORY STATS`
command.

additionally, clear scripts and stats between tests in external mode
(which is related to how this issue was discovered)
parent 5b84dc96
......@@ -44,6 +44,9 @@
"lua.caches": {
"type": "integer"
},
"script.VMs": {
"type": "integer"
},
"functions.caches": {
"type": "integer"
},
......
......@@ -730,7 +730,7 @@ NULL
}
}
unsigned long evalMemory(void) {
unsigned long evalScriptsMemoryVM(void) {
return luaMemory(lctx.lua);
}
......@@ -738,7 +738,7 @@ dict* evalScriptsDict(void) {
return lctx.lua_scripts;
}
unsigned long evalScriptsMemory(void) {
unsigned long evalScriptsMemoryEngine(void) {
return lctx.lua_scripts_mem +
dictMemUsage(lctx.lua_scripts) +
dictSize(lctx.lua_scripts) * sizeof(luaScript) +
......
......@@ -1063,7 +1063,7 @@ void functionLoadCommand(client *c) {
}
/* Return memory usage of all the engines combine */
unsigned long functionsMemory(void) {
unsigned long functionsMemoryVM(void) {
dictIterator *iter = dictGetIterator(engines);
dictEntry *entry = NULL;
size_t engines_memory = 0;
......@@ -1078,7 +1078,7 @@ unsigned long functionsMemory(void) {
}
/* Return memory overhead of all the engines combine */
unsigned long functionsMemoryOverhead(void) {
unsigned long functionsMemoryEngine(void) {
size_t memory_overhead = dictMemUsage(engines);
memory_overhead += dictMemUsage(curr_functions_lib_ctx->functions);
memory_overhead += sizeof(functionsLibCtx);
......
......@@ -102,8 +102,8 @@ struct functionLibInfo {
int functionsRegisterEngine(const char *engine_name, engine *engine_ctx);
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx, size_t timeout);
unsigned long functionsMemory(void);
unsigned long functionsMemoryOverhead(void);
unsigned long functionsMemoryVM(void);
unsigned long functionsMemoryEngine(void);
unsigned long functionsNum(void);
unsigned long functionsLibNum(void);
dict* functionsLibGet(void);
......
......@@ -1230,12 +1230,16 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
mh->aof_buffer = mem;
mem_total+=mem;
mem = evalScriptsMemory();
mh->lua_caches = mem;
mem = evalScriptsMemoryEngine();
mh->eval_caches = mem;
mem_total+=mem;
mh->functions_caches = functionsMemoryOverhead();
mh->functions_caches = functionsMemoryEngine();
mem_total+=mh->functions_caches;
mh->script_vm = evalScriptsMemoryVM();
mh->script_vm += functionsMemoryVM();
mem_total+=mh->script_vm;
for (j = 0; j < server.dbnum; j++) {
redisDb *db = server.db+j;
if (!kvstoreNumAllocatedDicts(db->keys)) continue;
......@@ -1556,7 +1560,7 @@ NULL
} else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) {
struct redisMemOverhead *mh = getMemoryOverheadData();
addReplyMapLen(c,31+mh->num_dbs);
addReplyMapLen(c,32+mh->num_dbs);
addReplyBulkCString(c,"peak.allocated");
addReplyLongLong(c,mh->peak_allocated);
......@@ -1583,11 +1587,14 @@ NULL
addReplyLongLong(c,mh->aof_buffer);
addReplyBulkCString(c,"lua.caches");
addReplyLongLong(c,mh->lua_caches);
addReplyLongLong(c,mh->eval_caches);
addReplyBulkCString(c,"functions.caches");
addReplyLongLong(c,mh->functions_caches);
addReplyBulkCString(c,"script.VMs");
addReplyLongLong(c,mh->script_vm);
for (size_t j = 0; j < mh->num_dbs; j++) {
char dbname[32];
snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid);
......
......@@ -5700,8 +5700,8 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
size_t zmalloc_used = zmalloc_used_memory();
size_t total_system_mem = server.system_memory_size;
const char *evict_policy = evictPolicyToString();
long long memory_lua = evalMemory();
long long memory_functions = functionsMemory();
long long memory_lua = evalScriptsMemoryVM();
long long memory_functions = functionsMemoryVM();
struct redisMemOverhead *mh = getMemoryOverheadData();
/* Peak memory is updated from time to time by serverCron() so it
......@@ -5716,7 +5716,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
bytesToHuman(total_system_hmem,sizeof(total_system_hmem),total_system_mem);
bytesToHuman(used_memory_lua_hmem,sizeof(used_memory_lua_hmem),memory_lua);
bytesToHuman(used_memory_vm_total_hmem,sizeof(used_memory_vm_total_hmem),memory_functions + memory_lua);
bytesToHuman(used_memory_scripts_hmem,sizeof(used_memory_scripts_hmem),mh->lua_caches + mh->functions_caches);
bytesToHuman(used_memory_scripts_hmem,sizeof(used_memory_scripts_hmem),mh->eval_caches + mh->functions_caches);
bytesToHuman(used_memory_rss_hmem,sizeof(used_memory_rss_hmem),server.cron_malloc_stats.process_rss);
bytesToHuman(maxmemory_hmem,sizeof(maxmemory_hmem),server.maxmemory);
......@@ -5742,7 +5742,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
"used_memory_lua:%lld\r\n", memory_lua, /* deprecated, renamed to used_memory_vm_eval */
"used_memory_vm_eval:%lld\r\n", memory_lua,
"used_memory_lua_human:%s\r\n", used_memory_lua_hmem, /* deprecated */
"used_memory_scripts_eval:%lld\r\n", (long long)mh->lua_caches,
"used_memory_scripts_eval:%lld\r\n", (long long)mh->eval_caches,
"number_of_cached_scripts:%lu\r\n", dictSize(evalScriptsDict()),
"number_of_functions:%lu\r\n", functionsNum(),
"number_of_libraries:%lu\r\n", functionsLibNum(),
......@@ -5750,7 +5750,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
"used_memory_vm_total:%lld\r\n", memory_functions + memory_lua,
"used_memory_vm_total_human:%s\r\n", used_memory_vm_total_hmem,
"used_memory_functions:%lld\r\n", (long long)mh->functions_caches,
"used_memory_scripts:%lld\r\n", (long long)mh->lua_caches + (long long)mh->functions_caches,
"used_memory_scripts:%lld\r\n", (long long)mh->eval_caches + (long long)mh->functions_caches,
"used_memory_scripts_human:%s\r\n", used_memory_scripts_hmem,
"maxmemory:%lld\r\n", server.maxmemory,
"maxmemory_human:%s\r\n", maxmemory_hmem,
......
......@@ -1401,8 +1401,9 @@ struct redisMemOverhead {
size_t clients_normal;
size_t cluster_links;
size_t aof_buffer;
size_t lua_caches;
size_t eval_caches;
size_t functions_caches;
size_t script_vm;
size_t overhead_total;
size_t dataset;
size_t total_keys;
......@@ -3510,9 +3511,9 @@ int ldbIsEnabled(void);
void ldbLog(sds entry);
void ldbLogRedisReply(char *reply);
void sha1hex(char *digest, char *script, size_t len);
unsigned long evalMemory(void);
unsigned long evalScriptsMemoryVM(void);
dict* evalScriptsDict(void);
unsigned long evalScriptsMemory(void);
unsigned long evalScriptsMemoryEngine(void);
uint64_t evalGetCommandFlags(client *c, uint64_t orig_flags);
uint64_t fcallGetCommandFlags(client *c, uint64_t orig_flags);
int isInsideYieldingLongCommand(void);
......
......@@ -373,6 +373,8 @@ proc run_external_server_test {code overrides} {
r flushall
r function flush
r script flush
r config resetstat
# store configs
set saved_config {}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment