Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
4ef8de8a
Commit
4ef8de8a
authored
Jan 05, 2010
by
antirez
Browse files
basic VM mostly working!
parent
d894161b
Changes
3
Hide whitespace changes
Inline
Side-by-side
redis.c
View file @
4ef8de8a
...
...
@@ -361,7 +361,7 @@ struct redisServer {
redisClient *master; /* client that is master for this slave */
int replstate;
unsigned int maxclients;
unsigned long maxmemory;
unsigned long
long
maxmemory;
unsigned int blockedclients;
/* Sort parameters - qsort_r() is only available under BSD so we
* have to take this state global, in order to pass it to sortCompare() */
...
...
@@ -372,7 +372,7 @@ struct redisServer {
int vm_enabled;
off_t vm_page_size;
off_t vm_pages;
long vm_max_memory;
unsigned long
long vm_max_memory;
/* Virtual memory state */
FILE *vm_fp;
int vm_fd;
...
...
@@ -459,6 +459,7 @@ static void addReplySds(redisClient *c, sds s);
static void incrRefCount(robj *o);
static int rdbSaveBackground(char *filename);
static robj *createStringObject(char *ptr, size_t len);
static robj *dupStringObject(robj *o);
static void replicationFeedSlaves(list *slaves, struct redisCommand *cmd, int dictid, robj **argv, int argc);
static void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc);
static int syncWithMaster(void);
...
...
@@ -491,6 +492,7 @@ static int handleClientsWaitingListPush(redisClient *c, robj *key, robj *ele);
static void vmInit(void);
static void vmMarkPagesFree(off_t page, off_t count);
static robj *vmLoadObject(robj *key);
static int vmSwapOneObject(void);
static void authCommand(redisClient *c);
static void pingCommand(redisClient *c);
...
...
@@ -1153,7 +1155,7 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD
/* Continue to expire if at the end of the cycle more than 25%
* of the keys were expired. */
do {
int
num = dictSize(db->expires);
long
num = dictSize(db->expires);
time_t now = time(NULL);
expired = 0;
...
...
@@ -1173,6 +1175,15 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD
} while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4);
}
/* Swap a few keys on disk if we are over the memory limit and VM
* is enbled. */
while (server.vm_enabled && zmalloc_used_memory() > server.vm_max_memory) {
if (vmSwapOneObject() == REDIS_ERR) {
redisLog(REDIS_WARNING,"WARNING: vm-max-memory limit reached but unable to swap more objects out!");
break;
}
}
/* Check if we should connect to a MASTER */
if (server.replstate == REDIS_REPL_CONNECT) {
redisLog(REDIS_NOTICE,"Connecting to MASTER...");
...
...
@@ -1502,6 +1513,12 @@ static void loadServerConfig(char *filename) {
if ((server.vm_enabled = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"vm-max-memory") && argc == 2) {
server.vm_max_memory = strtoll(argv[1], NULL, 10);
} else if (!strcasecmp(argv[0],"vm-page-size") && argc == 2) {
server.vm_page_size = strtoll(argv[1], NULL, 10);
} else if (!strcasecmp(argv[0],"vm-pages") && argc == 2) {
server.vm_pages = strtoll(argv[1], NULL, 10);
} else {
err = "Bad directive or wrong number of arguments"; goto loaderr;
}
...
...
@@ -2278,6 +2295,10 @@ static robj *createStringObject(char *ptr, size_t len) {
return createObject(REDIS_STRING,sdsnewlen(ptr,len));
}
static robj *dupStringObject(robj *o) {
return createStringObject(o->ptr,sdslen(o->ptr));
}
static robj *createListObject(void) {
list *l = listCreate();
...
...
@@ -6656,6 +6677,8 @@ static void vmInit(void) {
redisLog(REDIS_NOTICE,"Swap file allocated with success");
}
server.vm_bitmap = zmalloc((server.vm_pages+7)/8);
redisLog(REDIS_DEBUG,"Allocated %lld bytes page table for %lld pages",
(long long) (server.vm_pages+7)/8, server.vm_pages);
memset(server.vm_bitmap,0,(server.vm_pages+7)/8);
/* Try to remove the swap file, so the OS will really delete it from the
* file system when Redis exists. */
...
...
@@ -6667,7 +6690,8 @@ static void vmMarkPageUsed(off_t page) {
off_t byte = page/8;
int bit = page&7;
server.vm_bitmap[byte] |= 1<<bit;
printf("Mark used: %lld (byte:%d bit:%d)\n", (long long)page, byte, bit);
printf("Mark used: %lld (byte:%lld bit:%d)\n", (long long)page,
(long long)byte, bit);
}
/* Mark N contiguous pages as used, with 'page' being the first. */
...
...
@@ -6782,6 +6806,7 @@ static int vmSwapObject(robj *key, robj *val) {
off_t page;
assert(key->storage == REDIS_VM_MEMORY);
assert(key->refcount == 1);
if (vmFindContiguousPages(&page,pages) == REDIS_ERR) return REDIS_ERR;
if (fseeko(server.vm_fp,page*server.vm_page_size,SEEK_SET) == -1) {
redisLog(REDIS_WARNING,
...
...
@@ -6827,6 +6852,123 @@ static robj *vmLoadObject(robj *key) {
return val;
}
/* How a good candidate is this object for swapping?
* The better candidate it is, the greater the returned value.
*
* Currently we try to perform a fast estimation of the object size in
* memory, and combine it with aging informations.
*
* Basically swappability = idle-time * log(estimated size)
*
* Bigger objects are preferred over smaller objects, but not
* proportionally, this is why we use the logarithm. This algorithm is
* just a first try and will probably be tuned later. */
static double computeObjectSwappability(robj *o) {
time_t age = server.unixtime - o->vm.atime;
long asize = 0;
list *l;
dict *d;
struct dictEntry *de;
int z;
if (age <= 0) return 0;
switch(o->type) {
case REDIS_STRING:
if (o->encoding != REDIS_ENCODING_RAW) {
asize = sizeof(*o);
} else {
asize = sdslen(o->ptr)+sizeof(*o)+sizeof(long)*2;
}
break;
case REDIS_LIST:
l = o->ptr;
listNode *ln = listFirst(l);
asize = sizeof(list);
if (ln) {
robj *ele = ln->value;
long elesize;
elesize = (ele->encoding == REDIS_ENCODING_RAW) ?
(sizeof(*o)+sdslen(ele->ptr)) :
sizeof(*o);
asize += (sizeof(listNode)+elesize)*listLength(l);
}
break;
case REDIS_SET:
case REDIS_ZSET:
z = (o->type == REDIS_ZSET);
d = z ? ((zset*)o->ptr)->dict : o->ptr;
asize = sizeof(dict)+(sizeof(struct dictEntry*)*dictSlots(d));
if (z) asize += sizeof(zset)-sizeof(dict);
if (dictSize(d)) {
long elesize;
robj *ele;
de = dictGetRandomKey(d);
ele = dictGetEntryKey(de);
elesize = (ele->encoding == REDIS_ENCODING_RAW) ?
(sizeof(*o)+sdslen(ele->ptr)) :
sizeof(*o);
asize += (sizeof(struct dictEntry)+elesize)*dictSize(d);
if (z) asize += sizeof(zskiplistNode)*dictSize(d);
}
break;
}
return (double)asize*log(1+asize);
}
/* Try to swap an object that's a good candidate for swapping.
* Returns REDIS_OK if the object was swapped, REDIS_ERR if it's not possible
* to swap any object at all. */
static int vmSwapOneObject(void) {
int j, i;
struct dictEntry *best = NULL;
double best_swappability = 0;
robj *key, *val;
for (j = 0; j < server.dbnum; j++) {
redisDb *db = server.db+j;
if (dictSize(db->dict) == 0) continue;
for (i = 0; i < 5; i++) {
dictEntry *de;
double swappability;
de = dictGetRandomKey(db->dict);
key = dictGetEntryKey(de);
val = dictGetEntryVal(de);
if (key->storage != REDIS_VM_MEMORY) continue;
swappability = computeObjectSwappability(val);
if (!best || swappability > best_swappability) {
best = de;
best_swappability = swappability;
}
}
}
if (best == NULL) return REDIS_ERR;
key = dictGetEntryKey(best);
val = dictGetEntryVal(best);
redisLog(REDIS_DEBUG,"Key with best swappability: %s, %f\n",
key->ptr, best_swappability);
/* Unshare the key if needed */
if (key->refcount > 1) {
robj *newkey = dupStringObject(key);
decrRefCount(key);
key = dictGetEntryKey(best) = newkey;
}
/* Swap it */
if (vmSwapObject(key,val) == REDIS_OK) {
dictGetEntryVal(best) = NULL;
return REDIS_OK;
} else {
return REDIS_ERR;
}
}
/* ================================= Debugging ============================== */
static void debugCommand(redisClient *c) {
...
...
@@ -6880,6 +7022,13 @@ static void debugCommand(redisClient *c) {
}
key = dictGetEntryKey(de);
val = dictGetEntryVal(de);
/* If the key is shared we want to create a copy */
if (key->refcount > 1) {
robj *newkey = dupStringObject(key);
decrRefCount(key);
key = dictGetEntryKey(de) = newkey;
}
/* Swap it */
if (key->storage != REDIS_VM_MEMORY) {
addReplySds(c,sdsnew("-ERR This key is not in memory\r\n"));
} else if (vmSwapObject(key,val) == REDIS_OK) {
...
...
redis.conf
View file @
4ef8de8a
...
...
@@ -166,7 +166,43 @@ appendfsync always
################################ VIRTUAL MEMORY ###############################
# Virtual Memory allows Redis to work with datasets bigger than the actual
# amount of RAM needed to hold the whole dataset in memory.
# In order to do so very used keys are taken in memory while the other keys
# are swapped into a swap file, similarly to what operating systems do
# with memory pages.
#
# To enable VM just set 'vm-enabled' to yes, and set the following three
# VM parameters accordingly to your needs.
vm
-
enabled
yes
# vm-enabled no
# vm-max-memory configures the VM to use at max the specified amount of
# RAM. Everything that deos not fit will be swapped on disk *if* possible, that
# is, if there is still enough contiguous space in the swap file.
vm
-
max
-
memory
10000000
# Redis swap files is split into pages. An object can be saved using multiple
# contiguous pages, but pages can't be shared between different objects.
# So if your page is too big, small objects swapped out on disk will waste
# a lot of space. If you page is too small, there is less space in the swap
# file (assuming you configured the same number of total swap file pages).
#
# If you use a lot of small objects, use a page size of 64 or 32 bytes.
# If you use a lot of big objects, use a bigger page size.
# If unsure, use the defualt :)
vm
-
page
-
size
256
# Number of total memory pages in the swap file.
# Given that the page table (a bitmap of free/used pages) is taken in memory,
# every 8 pages on disk will consume 1 byte of RAM.
#
# The total swap size is vm-page-size * vm-pages
#
# With the default of 256-bytes memory pages and 104857600 pages Redis will
# use a 25 GB swap file, that will use rougly 13 MB of RAM for the page table.
vm
-
pages
104857600
############################### ADVANCED CONFIG ###############################
...
...
staticsymbols.h
View file @
4ef8de8a
...
...
@@ -17,6 +17,7 @@ static struct redisFunctionSym symsTable[] = {
{
"call"
,(
unsigned
long
)
call
},
{
"closeTimedoutClients"
,(
unsigned
long
)
closeTimedoutClients
},
{
"compareStringObjects"
,(
unsigned
long
)
compareStringObjects
},
{
"computeObjectSwappability"
,(
unsigned
long
)
computeObjectSwappability
},
{
"createClient"
,(
unsigned
long
)
createClient
},
{
"createListObject"
,(
unsigned
long
)
createListObject
},
{
"createObject"
,(
unsigned
long
)
createObject
},
...
...
@@ -40,6 +41,7 @@ static struct redisFunctionSym symsTable[] = {
{
"dictRedisObjectDestructor"
,(
unsigned
long
)
dictRedisObjectDestructor
},
{
"dictVanillaFree"
,(
unsigned
long
)
dictVanillaFree
},
{
"dupClientReplyValue"
,(
unsigned
long
)
dupClientReplyValue
},
{
"dupStringObject"
,(
unsigned
long
)
dupStringObject
},
{
"echoCommand"
,(
unsigned
long
)
echoCommand
},
{
"execCommand"
,(
unsigned
long
)
execCommand
},
{
"existsCommand"
,(
unsigned
long
)
existsCommand
},
...
...
@@ -119,6 +121,7 @@ static struct redisFunctionSym symsTable[] = {
{
"rdbLoadIntegerObject"
,(
unsigned
long
)
rdbLoadIntegerObject
},
{
"rdbLoadLen"
,(
unsigned
long
)
rdbLoadLen
},
{
"rdbLoadLzfStringObject"
,(
unsigned
long
)
rdbLoadLzfStringObject
},
{
"rdbLoadObject"
,(
unsigned
long
)
rdbLoadObject
},
{
"rdbLoadStringObject"
,(
unsigned
long
)
rdbLoadStringObject
},
{
"rdbLoadTime"
,(
unsigned
long
)
rdbLoadTime
},
{
"rdbLoadType"
,(
unsigned
long
)
rdbLoadType
},
...
...
@@ -128,10 +131,13 @@ static struct redisFunctionSym symsTable[] = {
{
"rdbSaveDoubleValue"
,(
unsigned
long
)
rdbSaveDoubleValue
},
{
"rdbSaveLen"
,(
unsigned
long
)
rdbSaveLen
},
{
"rdbSaveLzfStringObject"
,(
unsigned
long
)
rdbSaveLzfStringObject
},
{
"rdbSaveObject"
,(
unsigned
long
)
rdbSaveObject
},
{
"rdbSaveStringObject"
,(
unsigned
long
)
rdbSaveStringObject
},
{
"rdbSaveStringObjectRaw"
,(
unsigned
long
)
rdbSaveStringObjectRaw
},
{
"rdbSaveTime"
,(
unsigned
long
)
rdbSaveTime
},
{
"rdbSaveType"
,(
unsigned
long
)
rdbSaveType
},
{
"rdbSavedObjectLen"
,(
unsigned
long
)
rdbSavedObjectLen
},
{
"rdbSavedObjectPages"
,(
unsigned
long
)
rdbSavedObjectPages
},
{
"rdbTryIntegerEncoding"
,(
unsigned
long
)
rdbTryIntegerEncoding
},
{
"readQueryFromClient"
,(
unsigned
long
)
readQueryFromClient
},
{
"redisLog"
,(
unsigned
long
)
redisLog
},
...
...
@@ -193,6 +199,16 @@ static struct redisFunctionSym symsTable[] = {
{
"typeCommand"
,(
unsigned
long
)
typeCommand
},
{
"unblockClient"
,(
unsigned
long
)
unblockClient
},
{
"updateSlavesWaitingBgsave"
,(
unsigned
long
)
updateSlavesWaitingBgsave
},
{
"vmFindContiguousPages"
,(
unsigned
long
)
vmFindContiguousPages
},
{
"vmFreePage"
,(
unsigned
long
)
vmFreePage
},
{
"vmInit"
,(
unsigned
long
)
vmInit
},
{
"vmLoadObject"
,(
unsigned
long
)
vmLoadObject
},
{
"vmMarkPageFree"
,(
unsigned
long
)
vmMarkPageFree
},
{
"vmMarkPageUsed"
,(
unsigned
long
)
vmMarkPageUsed
},
{
"vmMarkPagesFree"
,(
unsigned
long
)
vmMarkPagesFree
},
{
"vmMarkPagesUsed"
,(
unsigned
long
)
vmMarkPagesUsed
},
{
"vmSwapObject"
,(
unsigned
long
)
vmSwapObject
},
{
"vmSwapOneObject"
,(
unsigned
long
)
vmSwapOneObject
},
{
"yesnotoi"
,(
unsigned
long
)
yesnotoi
},
{
"zaddCommand"
,(
unsigned
long
)
zaddCommand
},
{
"zaddGenericCommand"
,(
unsigned
long
)
zaddGenericCommand
},
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment