Unverified Commit 79fd2558 authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Add Lua VM memory to memory overhead, now that it's part of zmalloc (#13660)

To complement the work done in #13133.
it added the script VMs memory to be counted as part of zmalloc, but
that means they
should be also counted as part of the non-value overhead.

this commit contains some refactoring to make variable names and
function names less confusing.
it also adds a new field named `script.VMs` into the `MEMORY STATS`
command.

additionally, clear scripts and stats between tests in external mode
(which is related to how this issue was discovered)
parent 5b84dc96
...@@ -44,6 +44,9 @@ ...@@ -44,6 +44,9 @@
"lua.caches": { "lua.caches": {
"type": "integer" "type": "integer"
}, },
"script.VMs": {
"type": "integer"
},
"functions.caches": { "functions.caches": {
"type": "integer" "type": "integer"
}, },
......
...@@ -730,7 +730,7 @@ NULL ...@@ -730,7 +730,7 @@ NULL
} }
} }
unsigned long evalMemory(void) { unsigned long evalScriptsMemoryVM(void) {
return luaMemory(lctx.lua); return luaMemory(lctx.lua);
} }
...@@ -738,7 +738,7 @@ dict* evalScriptsDict(void) { ...@@ -738,7 +738,7 @@ dict* evalScriptsDict(void) {
return lctx.lua_scripts; return lctx.lua_scripts;
} }
unsigned long evalScriptsMemory(void) { unsigned long evalScriptsMemoryEngine(void) {
return lctx.lua_scripts_mem + return lctx.lua_scripts_mem +
dictMemUsage(lctx.lua_scripts) + dictMemUsage(lctx.lua_scripts) +
dictSize(lctx.lua_scripts) * sizeof(luaScript) + dictSize(lctx.lua_scripts) * sizeof(luaScript) +
......
...@@ -1063,7 +1063,7 @@ void functionLoadCommand(client *c) { ...@@ -1063,7 +1063,7 @@ void functionLoadCommand(client *c) {
} }
/* Return memory usage of all the engines combine */ /* Return memory usage of all the engines combine */
unsigned long functionsMemory(void) { unsigned long functionsMemoryVM(void) {
dictIterator *iter = dictGetIterator(engines); dictIterator *iter = dictGetIterator(engines);
dictEntry *entry = NULL; dictEntry *entry = NULL;
size_t engines_memory = 0; size_t engines_memory = 0;
...@@ -1078,7 +1078,7 @@ unsigned long functionsMemory(void) { ...@@ -1078,7 +1078,7 @@ unsigned long functionsMemory(void) {
} }
/* Return memory overhead of all the engines combine */ /* Return memory overhead of all the engines combine */
unsigned long functionsMemoryOverhead(void) { unsigned long functionsMemoryEngine(void) {
size_t memory_overhead = dictMemUsage(engines); size_t memory_overhead = dictMemUsage(engines);
memory_overhead += dictMemUsage(curr_functions_lib_ctx->functions); memory_overhead += dictMemUsage(curr_functions_lib_ctx->functions);
memory_overhead += sizeof(functionsLibCtx); memory_overhead += sizeof(functionsLibCtx);
......
...@@ -102,8 +102,8 @@ struct functionLibInfo { ...@@ -102,8 +102,8 @@ struct functionLibInfo {
int functionsRegisterEngine(const char *engine_name, engine *engine_ctx); int functionsRegisterEngine(const char *engine_name, engine *engine_ctx);
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx, size_t timeout); sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx, size_t timeout);
unsigned long functionsMemory(void); unsigned long functionsMemoryVM(void);
unsigned long functionsMemoryOverhead(void); unsigned long functionsMemoryEngine(void);
unsigned long functionsNum(void); unsigned long functionsNum(void);
unsigned long functionsLibNum(void); unsigned long functionsLibNum(void);
dict* functionsLibGet(void); dict* functionsLibGet(void);
......
...@@ -1230,12 +1230,16 @@ struct redisMemOverhead *getMemoryOverheadData(void) { ...@@ -1230,12 +1230,16 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
mh->aof_buffer = mem; mh->aof_buffer = mem;
mem_total+=mem; mem_total+=mem;
mem = evalScriptsMemory(); mem = evalScriptsMemoryEngine();
mh->lua_caches = mem; mh->eval_caches = mem;
mem_total+=mem; mem_total+=mem;
mh->functions_caches = functionsMemoryOverhead(); mh->functions_caches = functionsMemoryEngine();
mem_total+=mh->functions_caches; mem_total+=mh->functions_caches;
mh->script_vm = evalScriptsMemoryVM();
mh->script_vm += functionsMemoryVM();
mem_total+=mh->script_vm;
for (j = 0; j < server.dbnum; j++) { for (j = 0; j < server.dbnum; j++) {
redisDb *db = server.db+j; redisDb *db = server.db+j;
if (!kvstoreNumAllocatedDicts(db->keys)) continue; if (!kvstoreNumAllocatedDicts(db->keys)) continue;
...@@ -1556,7 +1560,7 @@ NULL ...@@ -1556,7 +1560,7 @@ NULL
} else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) { } else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) {
struct redisMemOverhead *mh = getMemoryOverheadData(); struct redisMemOverhead *mh = getMemoryOverheadData();
addReplyMapLen(c,31+mh->num_dbs); addReplyMapLen(c,32+mh->num_dbs);
addReplyBulkCString(c,"peak.allocated"); addReplyBulkCString(c,"peak.allocated");
addReplyLongLong(c,mh->peak_allocated); addReplyLongLong(c,mh->peak_allocated);
...@@ -1583,11 +1587,14 @@ NULL ...@@ -1583,11 +1587,14 @@ NULL
addReplyLongLong(c,mh->aof_buffer); addReplyLongLong(c,mh->aof_buffer);
addReplyBulkCString(c,"lua.caches"); addReplyBulkCString(c,"lua.caches");
addReplyLongLong(c,mh->lua_caches); addReplyLongLong(c,mh->eval_caches);
addReplyBulkCString(c,"functions.caches"); addReplyBulkCString(c,"functions.caches");
addReplyLongLong(c,mh->functions_caches); addReplyLongLong(c,mh->functions_caches);
addReplyBulkCString(c,"script.VMs");
addReplyLongLong(c,mh->script_vm);
for (size_t j = 0; j < mh->num_dbs; j++) { for (size_t j = 0; j < mh->num_dbs; j++) {
char dbname[32]; char dbname[32];
snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid); snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid);
......
...@@ -5700,8 +5700,8 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) { ...@@ -5700,8 +5700,8 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
size_t zmalloc_used = zmalloc_used_memory(); size_t zmalloc_used = zmalloc_used_memory();
size_t total_system_mem = server.system_memory_size; size_t total_system_mem = server.system_memory_size;
const char *evict_policy = evictPolicyToString(); const char *evict_policy = evictPolicyToString();
long long memory_lua = evalMemory(); long long memory_lua = evalScriptsMemoryVM();
long long memory_functions = functionsMemory(); long long memory_functions = functionsMemoryVM();
struct redisMemOverhead *mh = getMemoryOverheadData(); struct redisMemOverhead *mh = getMemoryOverheadData();
/* Peak memory is updated from time to time by serverCron() so it /* Peak memory is updated from time to time by serverCron() so it
...@@ -5716,7 +5716,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) { ...@@ -5716,7 +5716,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
bytesToHuman(total_system_hmem,sizeof(total_system_hmem),total_system_mem); bytesToHuman(total_system_hmem,sizeof(total_system_hmem),total_system_mem);
bytesToHuman(used_memory_lua_hmem,sizeof(used_memory_lua_hmem),memory_lua); bytesToHuman(used_memory_lua_hmem,sizeof(used_memory_lua_hmem),memory_lua);
bytesToHuman(used_memory_vm_total_hmem,sizeof(used_memory_vm_total_hmem),memory_functions + memory_lua); bytesToHuman(used_memory_vm_total_hmem,sizeof(used_memory_vm_total_hmem),memory_functions + memory_lua);
bytesToHuman(used_memory_scripts_hmem,sizeof(used_memory_scripts_hmem),mh->lua_caches + mh->functions_caches); bytesToHuman(used_memory_scripts_hmem,sizeof(used_memory_scripts_hmem),mh->eval_caches + mh->functions_caches);
bytesToHuman(used_memory_rss_hmem,sizeof(used_memory_rss_hmem),server.cron_malloc_stats.process_rss); bytesToHuman(used_memory_rss_hmem,sizeof(used_memory_rss_hmem),server.cron_malloc_stats.process_rss);
bytesToHuman(maxmemory_hmem,sizeof(maxmemory_hmem),server.maxmemory); bytesToHuman(maxmemory_hmem,sizeof(maxmemory_hmem),server.maxmemory);
...@@ -5742,7 +5742,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) { ...@@ -5742,7 +5742,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
"used_memory_lua:%lld\r\n", memory_lua, /* deprecated, renamed to used_memory_vm_eval */ "used_memory_lua:%lld\r\n", memory_lua, /* deprecated, renamed to used_memory_vm_eval */
"used_memory_vm_eval:%lld\r\n", memory_lua, "used_memory_vm_eval:%lld\r\n", memory_lua,
"used_memory_lua_human:%s\r\n", used_memory_lua_hmem, /* deprecated */ "used_memory_lua_human:%s\r\n", used_memory_lua_hmem, /* deprecated */
"used_memory_scripts_eval:%lld\r\n", (long long)mh->lua_caches, "used_memory_scripts_eval:%lld\r\n", (long long)mh->eval_caches,
"number_of_cached_scripts:%lu\r\n", dictSize(evalScriptsDict()), "number_of_cached_scripts:%lu\r\n", dictSize(evalScriptsDict()),
"number_of_functions:%lu\r\n", functionsNum(), "number_of_functions:%lu\r\n", functionsNum(),
"number_of_libraries:%lu\r\n", functionsLibNum(), "number_of_libraries:%lu\r\n", functionsLibNum(),
...@@ -5750,7 +5750,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) { ...@@ -5750,7 +5750,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
"used_memory_vm_total:%lld\r\n", memory_functions + memory_lua, "used_memory_vm_total:%lld\r\n", memory_functions + memory_lua,
"used_memory_vm_total_human:%s\r\n", used_memory_vm_total_hmem, "used_memory_vm_total_human:%s\r\n", used_memory_vm_total_hmem,
"used_memory_functions:%lld\r\n", (long long)mh->functions_caches, "used_memory_functions:%lld\r\n", (long long)mh->functions_caches,
"used_memory_scripts:%lld\r\n", (long long)mh->lua_caches + (long long)mh->functions_caches, "used_memory_scripts:%lld\r\n", (long long)mh->eval_caches + (long long)mh->functions_caches,
"used_memory_scripts_human:%s\r\n", used_memory_scripts_hmem, "used_memory_scripts_human:%s\r\n", used_memory_scripts_hmem,
"maxmemory:%lld\r\n", server.maxmemory, "maxmemory:%lld\r\n", server.maxmemory,
"maxmemory_human:%s\r\n", maxmemory_hmem, "maxmemory_human:%s\r\n", maxmemory_hmem,
......
...@@ -1401,8 +1401,9 @@ struct redisMemOverhead { ...@@ -1401,8 +1401,9 @@ struct redisMemOverhead {
size_t clients_normal; size_t clients_normal;
size_t cluster_links; size_t cluster_links;
size_t aof_buffer; size_t aof_buffer;
size_t lua_caches; size_t eval_caches;
size_t functions_caches; size_t functions_caches;
size_t script_vm;
size_t overhead_total; size_t overhead_total;
size_t dataset; size_t dataset;
size_t total_keys; size_t total_keys;
...@@ -3510,9 +3511,9 @@ int ldbIsEnabled(void); ...@@ -3510,9 +3511,9 @@ int ldbIsEnabled(void);
void ldbLog(sds entry); void ldbLog(sds entry);
void ldbLogRedisReply(char *reply); void ldbLogRedisReply(char *reply);
void sha1hex(char *digest, char *script, size_t len); void sha1hex(char *digest, char *script, size_t len);
unsigned long evalMemory(void); unsigned long evalScriptsMemoryVM(void);
dict* evalScriptsDict(void); dict* evalScriptsDict(void);
unsigned long evalScriptsMemory(void); unsigned long evalScriptsMemoryEngine(void);
uint64_t evalGetCommandFlags(client *c, uint64_t orig_flags); uint64_t evalGetCommandFlags(client *c, uint64_t orig_flags);
uint64_t fcallGetCommandFlags(client *c, uint64_t orig_flags); uint64_t fcallGetCommandFlags(client *c, uint64_t orig_flags);
int isInsideYieldingLongCommand(void); int isInsideYieldingLongCommand(void);
......
...@@ -373,6 +373,8 @@ proc run_external_server_test {code overrides} { ...@@ -373,6 +373,8 @@ proc run_external_server_test {code overrides} {
r flushall r flushall
r function flush r function flush
r script flush
r config resetstat
# store configs # store configs
set saved_config {} set saved_config {}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment