Commit 884d4b39 authored by antirez's avatar antirez
Browse files

Prevent hash table resize while there are active child processes in order to...

Prevent hash table resize while there are active child processes in order to play well with copy on write
parent 5727b9aa
...@@ -45,6 +45,12 @@ ...@@ -45,6 +45,12 @@
#include "dict.h" #include "dict.h"
#include "zmalloc.h" #include "zmalloc.h"
/* Using dictEnableResize() / dictDisableResize() we make possible to
* enable/disable resizing of the hash table as needed. This is very important
* for Redis, as we use copy-on-write and don't want to move too much memory
* around when there is a child performing saving operations. */
static int dict_can_resize = 1;
/* ---------------------------- Utility funcitons --------------------------- */ /* ---------------------------- Utility funcitons --------------------------- */
static void _dictPanic(const char *fmt, ...) static void _dictPanic(const char *fmt, ...)
...@@ -147,6 +153,7 @@ int dictResize(dict *ht) ...@@ -147,6 +153,7 @@ int dictResize(dict *ht)
{ {
int minimal = ht->used; int minimal = ht->used;
if (!dict_can_resize) return DICT_ERR;
if (minimal < DICT_HT_INITIAL_SIZE) if (minimal < DICT_HT_INITIAL_SIZE)
minimal = DICT_HT_INITIAL_SIZE; minimal = DICT_HT_INITIAL_SIZE;
return dictExpand(ht, minimal); return dictExpand(ht, minimal);
...@@ -417,7 +424,7 @@ static int _dictExpandIfNeeded(dict *ht) ...@@ -417,7 +424,7 @@ static int _dictExpandIfNeeded(dict *ht)
* if the table is "full" dobule its size. */ * if the table is "full" dobule its size. */
if (ht->size == 0) if (ht->size == 0)
return dictExpand(ht, DICT_HT_INITIAL_SIZE); return dictExpand(ht, DICT_HT_INITIAL_SIZE);
if (ht->used == ht->size) if (ht->used >= ht->size && dict_can_resize)
return dictExpand(ht, ht->size*2); return dictExpand(ht, ht->size*2);
return DICT_OK; return DICT_OK;
} }
...@@ -507,6 +514,14 @@ void dictPrintStats(dict *ht) { ...@@ -507,6 +514,14 @@ void dictPrintStats(dict *ht) {
} }
} }
void dictEnableResize(void) {
dict_can_resize = 1;
}
void dictDisableResize(void) {
dict_can_resize = 0;
}
/* ----------------------- StringCopy Hash Table Type ------------------------*/ /* ----------------------- StringCopy Hash Table Type ------------------------*/
static unsigned int _dictStringCopyHTHashFunction(const void *key) static unsigned int _dictStringCopyHTHashFunction(const void *key)
......
...@@ -127,6 +127,8 @@ dictEntry *dictGetRandomKey(dict *ht); ...@@ -127,6 +127,8 @@ dictEntry *dictGetRandomKey(dict *ht);
void dictPrintStats(dict *ht); void dictPrintStats(dict *ht);
unsigned int dictGenHashFunction(const unsigned char *buf, int len); unsigned int dictGenHashFunction(const unsigned char *buf, int len);
void dictEmpty(dict *ht); void dictEmpty(dict *ht);
void dictEnableResize(void);
void dictDisableResize(void);
/* Hash table types */ /* Hash table types */
extern dictType dictTypeHeapStringCopyKey; extern dictType dictTypeHeapStringCopyKey;
......
...@@ -1294,6 +1294,19 @@ cleanup: ...@@ -1294,6 +1294,19 @@ cleanup:
server.bgrewritechildpid = -1; server.bgrewritechildpid = -1;
} }
/* This function is called once a background process of some kind terminates,
* as we want to avoid resizing the hash tables when there is a child in order
* to play well with copy-on-write (otherwise when a resize happens lots of
* memory pages are copied). The goal of this function is to update the ability
* for dict.c to resize the hash tables accordingly to the fact we have o not
* running childs. */
static void updateDictResizePolicy(void) {
if (server.bgsavechildpid == -1 && server.bgrewritechildpid == -1)
dictEnableResize();
else
dictDisableResize();
}
static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
int j, loops = server.cronloops++; int j, loops = server.cronloops++;
REDIS_NOTUSED(eventLoop); REDIS_NOTUSED(eventLoop);
...@@ -1325,7 +1338,11 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD ...@@ -1325,7 +1338,11 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD
* if we resize the HT while there is the saving child at work actually * if we resize the HT while there is the saving child at work actually
* a lot of memory movements in the parent will cause a lot of pages * a lot of memory movements in the parent will cause a lot of pages
* copied. */ * copied. */
if (server.bgsavechildpid == -1 && !(loops % 10)) tryResizeHashTables(); if (server.bgsavechildpid == -1 && server.bgrewritechildpid == -1 &&
!(loops % 10))
{
tryResizeHashTables();
}
/* Show information about connected clients */ /* Show information about connected clients */
if (!(loops % 50)) { if (!(loops % 50)) {
...@@ -1351,6 +1368,7 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD ...@@ -1351,6 +1368,7 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD
} else { } else {
backgroundRewriteDoneHandler(statloc); backgroundRewriteDoneHandler(statloc);
} }
updateDictResizePolicy();
} }
} else { } else {
/* If there is not a background saving in progress check if /* If there is not a background saving in progress check if
...@@ -3497,6 +3515,7 @@ static int rdbSaveBackground(char *filename) { ...@@ -3497,6 +3515,7 @@ static int rdbSaveBackground(char *filename) {
} }
redisLog(REDIS_NOTICE,"Background saving started by pid %d",childpid); redisLog(REDIS_NOTICE,"Background saving started by pid %d",childpid);
server.bgsavechildpid = childpid; server.bgsavechildpid = childpid;
updateDictResizePolicy();
return REDIS_OK; return REDIS_OK;
} }
return REDIS_OK; /* unreached */ return REDIS_OK; /* unreached */
...@@ -8116,6 +8135,7 @@ static int rewriteAppendOnlyFileBackground(void) { ...@@ -8116,6 +8135,7 @@ static int rewriteAppendOnlyFileBackground(void) {
redisLog(REDIS_NOTICE, redisLog(REDIS_NOTICE,
"Background append only file rewriting started by pid %d",childpid); "Background append only file rewriting started by pid %d",childpid);
server.bgrewritechildpid = childpid; server.bgrewritechildpid = childpid;
updateDictResizePolicy();
/* We set appendseldb to -1 in order to force the next call to the /* We set appendseldb to -1 in order to force the next call to the
* feedAppendOnlyFile() to issue a SELECT command, so the differences * feedAppendOnlyFile() to issue a SELECT command, so the differences
* accumulated by the parent into server.bgrewritebuf will start * accumulated by the parent into server.bgrewritebuf will start
......
...@@ -8,6 +8,7 @@ static struct redisFunctionSym symsTable[] = { ...@@ -8,6 +8,7 @@ static struct redisFunctionSym symsTable[] = {
{"addReplyBulkLen",(unsigned long)addReplyBulkLen}, {"addReplyBulkLen",(unsigned long)addReplyBulkLen},
{"addReplyDouble",(unsigned long)addReplyDouble}, {"addReplyDouble",(unsigned long)addReplyDouble},
{"addReplyLong",(unsigned long)addReplyLong}, {"addReplyLong",(unsigned long)addReplyLong},
{"addReplyLongLong",(unsigned long)addReplyLongLong},
{"addReplySds",(unsigned long)addReplySds}, {"addReplySds",(unsigned long)addReplySds},
{"addReplyUlong",(unsigned long)addReplyUlong}, {"addReplyUlong",(unsigned long)addReplyUlong},
{"aofRemoveTempFile",(unsigned long)aofRemoveTempFile}, {"aofRemoveTempFile",(unsigned long)aofRemoveTempFile},
...@@ -80,6 +81,7 @@ static struct redisFunctionSym symsTable[] = { ...@@ -80,6 +81,7 @@ static struct redisFunctionSym symsTable[] = {
{"freeIOJob",(unsigned long)freeIOJob}, {"freeIOJob",(unsigned long)freeIOJob},
{"freeListObject",(unsigned long)freeListObject}, {"freeListObject",(unsigned long)freeListObject},
{"freeMemoryIfNeeded",(unsigned long)freeMemoryIfNeeded}, {"freeMemoryIfNeeded",(unsigned long)freeMemoryIfNeeded},
{"freePubsubPattern",(unsigned long)freePubsubPattern},
{"freeSetObject",(unsigned long)freeSetObject}, {"freeSetObject",(unsigned long)freeSetObject},
{"freeStringObject",(unsigned long)freeStringObject}, {"freeStringObject",(unsigned long)freeStringObject},
{"freeZsetObject",(unsigned long)freeZsetObject}, {"freeZsetObject",(unsigned long)freeZsetObject},
...@@ -103,6 +105,7 @@ static struct redisFunctionSym symsTable[] = { ...@@ -103,6 +105,7 @@ static struct redisFunctionSym symsTable[] = {
{"hexistsCommand",(unsigned long)hexistsCommand}, {"hexistsCommand",(unsigned long)hexistsCommand},
{"hgetCommand",(unsigned long)hgetCommand}, {"hgetCommand",(unsigned long)hgetCommand},
{"hgetallCommand",(unsigned long)hgetallCommand}, {"hgetallCommand",(unsigned long)hgetallCommand},
{"hincrbyCommand",(unsigned long)hincrbyCommand},
{"hkeysCommand",(unsigned long)hkeysCommand}, {"hkeysCommand",(unsigned long)hkeysCommand},
{"hlenCommand",(unsigned long)hlenCommand}, {"hlenCommand",(unsigned long)hlenCommand},
{"hsetCommand",(unsigned long)hsetCommand}, {"hsetCommand",(unsigned long)hsetCommand},
...@@ -120,6 +123,8 @@ static struct redisFunctionSym symsTable[] = { ...@@ -120,6 +123,8 @@ static struct redisFunctionSym symsTable[] = {
{"keysCommand",(unsigned long)keysCommand}, {"keysCommand",(unsigned long)keysCommand},
{"lastsaveCommand",(unsigned long)lastsaveCommand}, {"lastsaveCommand",(unsigned long)lastsaveCommand},
{"lindexCommand",(unsigned long)lindexCommand}, {"lindexCommand",(unsigned long)lindexCommand},
{"listMatchObjects",(unsigned long)listMatchObjects},
{"listMatchPubsubPattern",(unsigned long)listMatchPubsubPattern},
{"llenCommand",(unsigned long)llenCommand}, {"llenCommand",(unsigned long)llenCommand},
{"loadServerConfig",(unsigned long)loadServerConfig}, {"loadServerConfig",(unsigned long)loadServerConfig},
{"lockThreadedIO",(unsigned long)lockThreadedIO}, {"lockThreadedIO",(unsigned long)lockThreadedIO},
...@@ -147,6 +152,16 @@ static struct redisFunctionSym symsTable[] = { ...@@ -147,6 +152,16 @@ static struct redisFunctionSym symsTable[] = {
{"popGenericCommand",(unsigned long)popGenericCommand}, {"popGenericCommand",(unsigned long)popGenericCommand},
{"processCommand",(unsigned long)processCommand}, {"processCommand",(unsigned long)processCommand},
{"processInputBuffer",(unsigned long)processInputBuffer}, {"processInputBuffer",(unsigned long)processInputBuffer},
{"psubscribeCommand",(unsigned long)psubscribeCommand},
{"publishCommand",(unsigned long)publishCommand},
{"pubsubPublishMessage",(unsigned long)pubsubPublishMessage},
{"pubsubSubscribeChannel",(unsigned long)pubsubSubscribeChannel},
{"pubsubSubscribePattern",(unsigned long)pubsubSubscribePattern},
{"pubsubUnsubscribeAllChannels",(unsigned long)pubsubUnsubscribeAllChannels},
{"pubsubUnsubscribeAllPatterns",(unsigned long)pubsubUnsubscribeAllPatterns},
{"pubsubUnsubscribeChannel",(unsigned long)pubsubUnsubscribeChannel},
{"pubsubUnsubscribePattern",(unsigned long)pubsubUnsubscribePattern},
{"punsubscribeCommand",(unsigned long)punsubscribeCommand},
{"pushGenericCommand",(unsigned long)pushGenericCommand}, {"pushGenericCommand",(unsigned long)pushGenericCommand},
{"qsortCompareSetsByCardinality",(unsigned long)qsortCompareSetsByCardinality}, {"qsortCompareSetsByCardinality",(unsigned long)qsortCompareSetsByCardinality},
{"qsortCompareZsetopsrcByCardinality",(unsigned long)qsortCompareZsetopsrcByCardinality}, {"qsortCompareZsetopsrcByCardinality",(unsigned long)qsortCompareZsetopsrcByCardinality},
...@@ -224,6 +239,7 @@ static struct redisFunctionSym symsTable[] = { ...@@ -224,6 +239,7 @@ static struct redisFunctionSym symsTable[] = {
{"stringObjectLen",(unsigned long)stringObjectLen}, {"stringObjectLen",(unsigned long)stringObjectLen},
{"stringmatch",(unsigned long)stringmatch}, {"stringmatch",(unsigned long)stringmatch},
{"stringmatchlen",(unsigned long)stringmatchlen}, {"stringmatchlen",(unsigned long)stringmatchlen},
{"subscribeCommand",(unsigned long)subscribeCommand},
{"substrCommand",(unsigned long)substrCommand}, {"substrCommand",(unsigned long)substrCommand},
{"sunionCommand",(unsigned long)sunionCommand}, {"sunionCommand",(unsigned long)sunionCommand},
{"sunionDiffGenericCommand",(unsigned long)sunionDiffGenericCommand}, {"sunionDiffGenericCommand",(unsigned long)sunionDiffGenericCommand},
...@@ -241,6 +257,8 @@ static struct redisFunctionSym symsTable[] = { ...@@ -241,6 +257,8 @@ static struct redisFunctionSym symsTable[] = {
{"typeCommand",(unsigned long)typeCommand}, {"typeCommand",(unsigned long)typeCommand},
{"unblockClientWaitingData",(unsigned long)unblockClientWaitingData}, {"unblockClientWaitingData",(unsigned long)unblockClientWaitingData},
{"unlockThreadedIO",(unsigned long)unlockThreadedIO}, {"unlockThreadedIO",(unsigned long)unlockThreadedIO},
{"unsubscribeCommand",(unsigned long)unsubscribeCommand},
{"updateDictResizePolicy",(unsigned long)updateDictResizePolicy},
{"updateSlavesWaitingBgsave",(unsigned long)updateSlavesWaitingBgsave}, {"updateSlavesWaitingBgsave",(unsigned long)updateSlavesWaitingBgsave},
{"usage",(unsigned long)usage}, {"usage",(unsigned long)usage},
{"version",(unsigned long)version}, {"version",(unsigned long)version},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment