Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
4ef8de8a
Commit
4ef8de8a
authored
Jan 05, 2010
by
antirez
Browse files
basic VM mostly working!
parent
d894161b
Changes
3
Hide whitespace changes
Inline
Side-by-side
redis.c
View file @
4ef8de8a
...
...
@@ -361,7 +361,7 @@ struct redisServer {
redisClient *master; /* client that is master for this slave */
int replstate;
unsigned int maxclients;
unsigned long maxmemory;
unsigned long
long
maxmemory;
unsigned int blockedclients;
/* Sort parameters - qsort_r() is only available under BSD so we
* have to take this state global, in order to pass it to sortCompare() */
...
...
@@ -372,7 +372,7 @@ struct redisServer {
int vm_enabled;
off_t vm_page_size;
off_t vm_pages;
long vm_max_memory;
unsigned long
long vm_max_memory;
/* Virtual memory state */
FILE *vm_fp;
int vm_fd;
...
...
@@ -459,6 +459,7 @@ static void addReplySds(redisClient *c, sds s);
static void incrRefCount(robj *o);
static int rdbSaveBackground(char *filename);
static robj *createStringObject(char *ptr, size_t len);
static robj *dupStringObject(robj *o);
static void replicationFeedSlaves(list *slaves, struct redisCommand *cmd, int dictid, robj **argv, int argc);
static void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc);
static int syncWithMaster(void);
...
...
@@ -491,6 +492,7 @@ static int handleClientsWaitingListPush(redisClient *c, robj *key, robj *ele);
static void vmInit(void);
static void vmMarkPagesFree(off_t page, off_t count);
static robj *vmLoadObject(robj *key);
static int vmSwapOneObject(void);
static void authCommand(redisClient *c);
static void pingCommand(redisClient *c);
...
...
@@ -1153,7 +1155,7 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD
/* Continue to expire if at the end of the cycle more than 25%
* of the keys were expired. */
do {
int
num = dictSize(db->expires);
long
num = dictSize(db->expires);
time_t now = time(NULL);
expired = 0;
...
...
@@ -1173,6 +1175,15 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD
} while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4);
}
/* Swap a few keys on disk if we are over the memory limit and VM
* is enbled. */
while (server.vm_enabled && zmalloc_used_memory() > server.vm_max_memory) {
if (vmSwapOneObject() == REDIS_ERR) {
redisLog(REDIS_WARNING,"WARNING: vm-max-memory limit reached but unable to swap more objects out!");
break;
}
}
/* Check if we should connect to a MASTER */
if (server.replstate == REDIS_REPL_CONNECT) {
redisLog(REDIS_NOTICE,"Connecting to MASTER...");
...
...
@@ -1502,6 +1513,12 @@ static void loadServerConfig(char *filename) {
if ((server.vm_enabled = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"vm-max-memory") && argc == 2) {
server.vm_max_memory = strtoll(argv[1], NULL, 10);
} else if (!strcasecmp(argv[0],"vm-page-size") && argc == 2) {
server.vm_page_size = strtoll(argv[1], NULL, 10);
} else if (!strcasecmp(argv[0],"vm-pages") && argc == 2) {
server.vm_pages = strtoll(argv[1], NULL, 10);
} else {
err = "Bad directive or wrong number of arguments"; goto loaderr;
}
...
...
@@ -2278,6 +2295,10 @@ static robj *createStringObject(char *ptr, size_t len) {
return createObject(REDIS_STRING,sdsnewlen(ptr,len));
}
static robj *dupStringObject(robj *o) {
return createStringObject(o->ptr,sdslen(o->ptr));
}
static robj *createListObject(void) {
list *l = listCreate();
...
...
@@ -6656,6 +6677,8 @@ static void vmInit(void) {
redisLog(REDIS_NOTICE,"Swap file allocated with success");
}
server.vm_bitmap = zmalloc((server.vm_pages+7)/8);
redisLog(REDIS_DEBUG,"Allocated %lld bytes page table for %lld pages",
(long long) (server.vm_pages+7)/8, server.vm_pages);
memset(server.vm_bitmap,0,(server.vm_pages+7)/8);
/* Try to remove the swap file, so the OS will really delete it from the
* file system when Redis exists. */
...
...
@@ -6667,7 +6690,8 @@ static void vmMarkPageUsed(off_t page) {
off_t byte = page/8;
int bit = page&7;
server.vm_bitmap[byte] |= 1<<bit;
printf("Mark used: %lld (byte:%d bit:%d)\n", (long long)page, byte, bit);
printf("Mark used: %lld (byte:%lld bit:%d)\n", (long long)page,
(long long)byte, bit);
}
/* Mark N contiguous pages as used, with 'page' being the first. */
...
...
@@ -6782,6 +6806,7 @@ static int vmSwapObject(robj *key, robj *val) {
off_t page;
assert(key->storage == REDIS_VM_MEMORY);
assert(key->refcount == 1);
if (vmFindContiguousPages(&page,pages) == REDIS_ERR) return REDIS_ERR;
if (fseeko(server.vm_fp,page*server.vm_page_size,SEEK_SET) == -1) {
redisLog(REDIS_WARNING,
...
...
@@ -6827,6 +6852,123 @@ static robj *vmLoadObject(robj *key) {
return val;
}
/* How a good candidate is this object for swapping?
* The better candidate it is, the greater the returned value.
*
* Currently we try to perform a fast estimation of the object size in
* memory, and combine it with aging informations.
*
* Basically swappability = idle-time * log(estimated size)
*
* Bigger objects are preferred over smaller objects, but not
* proportionally, this is why we use the logarithm. This algorithm is
* just a first try and will probably be tuned later. */
static double computeObjectSwappability(robj *o) {
time_t age = server.unixtime - o->vm.atime;
long asize = 0;
list *l;
dict *d;
struct dictEntry *de;
int z;
if (age <= 0) return 0;
switch(o->type) {
case REDIS_STRING:
if (o->encoding != REDIS_ENCODING_RAW) {
asize = sizeof(*o);
} else {
asize = sdslen(o->ptr)+sizeof(*o)+sizeof(long)*2;
}
break;
case REDIS_LIST:
l = o->ptr;
listNode *ln = listFirst(l);
asize = sizeof(list);
if (ln) {
robj *ele = ln->value;
long elesize;
elesize = (ele->encoding == REDIS_ENCODING_RAW) ?
(sizeof(*o)+sdslen(ele->ptr)) :
sizeof(*o);
asize += (sizeof(listNode)+elesize)*listLength(l);
}
break;
case REDIS_SET:
case REDIS_ZSET:
z = (o->type == REDIS_ZSET);
d = z ? ((zset*)o->ptr)->dict : o->ptr;
asize = sizeof(dict)+(sizeof(struct dictEntry*)*dictSlots(d));
if (z) asize += sizeof(zset)-sizeof(dict);
if (dictSize(d)) {
long elesize;
robj *ele;
de = dictGetRandomKey(d);
ele = dictGetEntryKey(de);
elesize = (ele->encoding == REDIS_ENCODING_RAW) ?
(sizeof(*o)+sdslen(ele->ptr)) :
sizeof(*o);
asize += (sizeof(struct dictEntry)+elesize)*dictSize(d);
if (z) asize += sizeof(zskiplistNode)*dictSize(d);
}
break;
}
return (double)asize*log(1+asize);
}
/* Try to swap an object that's a good candidate for swapping.
* Returns REDIS_OK if the object was swapped, REDIS_ERR if it's not possible
* to swap any object at all. */
static int vmSwapOneObject(void) {
int j, i;
struct dictEntry *best = NULL;
double best_swappability = 0;
robj *key, *val;
for (j = 0; j < server.dbnum; j++) {
redisDb *db = server.db+j;
if (dictSize(db->dict) == 0) continue;
for (i = 0; i < 5; i++) {
dictEntry *de;
double swappability;
de = dictGetRandomKey(db->dict);
key = dictGetEntryKey(de);
val = dictGetEntryVal(de);
if (key->storage != REDIS_VM_MEMORY) continue;
swappability = computeObjectSwappability(val);
if (!best || swappability > best_swappability) {
best = de;
best_swappability = swappability;
}
}
}
if (best == NULL) return REDIS_ERR;
key = dictGetEntryKey(best);
val = dictGetEntryVal(best);
redisLog(REDIS_DEBUG,"Key with best swappability: %s, %f\n",
key->ptr, best_swappability);
/* Unshare the key if needed */
if (key->refcount > 1) {
robj *newkey = dupStringObject(key);
decrRefCount(key);
key = dictGetEntryKey(best) = newkey;
}
/* Swap it */
if (vmSwapObject(key,val) == REDIS_OK) {
dictGetEntryVal(best) = NULL;
return REDIS_OK;
} else {
return REDIS_ERR;
}
}
/* ================================= Debugging ============================== */
static void debugCommand(redisClient *c) {
...
...
@@ -6880,6 +7022,13 @@ static void debugCommand(redisClient *c) {
}
key = dictGetEntryKey(de);
val = dictGetEntryVal(de);
/* If the key is shared we want to create a copy */
if (key->refcount > 1) {
robj *newkey = dupStringObject(key);
decrRefCount(key);
key = dictGetEntryKey(de) = newkey;
}
/* Swap it */
if (key->storage != REDIS_VM_MEMORY) {
addReplySds(c,sdsnew("-ERR This key is not in memory\r\n"));
} else if (vmSwapObject(key,val) == REDIS_OK) {
...
...
redis.conf
View file @
4ef8de8a
...
...
@@ -166,7 +166,43 @@ appendfsync always
################################ VIRTUAL MEMORY ###############################
# Virtual Memory allows Redis to work with datasets bigger than the actual
# amount of RAM needed to hold the whole dataset in memory.
# In order to do so very used keys are taken in memory while the other keys
# are swapped into a swap file, similarly to what operating systems do
# with memory pages.
#
# To enable VM just set 'vm-enabled' to yes, and set the following three
# VM parameters accordingly to your needs.
vm
-
enabled
yes
# vm-enabled no
# vm-max-memory configures the VM to use at max the specified amount of
# RAM. Everything that deos not fit will be swapped on disk *if* possible, that
# is, if there is still enough contiguous space in the swap file.
vm
-
max
-
memory
10000000
# Redis swap files is split into pages. An object can be saved using multiple
# contiguous pages, but pages can't be shared between different objects.
# So if your page is too big, small objects swapped out on disk will waste
# a lot of space. If you page is too small, there is less space in the swap
# file (assuming you configured the same number of total swap file pages).
#
# If you use a lot of small objects, use a page size of 64 or 32 bytes.
# If you use a lot of big objects, use a bigger page size.
# If unsure, use the defualt :)
vm
-
page
-
size
256
# Number of total memory pages in the swap file.
# Given that the page table (a bitmap of free/used pages) is taken in memory,
# every 8 pages on disk will consume 1 byte of RAM.
#
# The total swap size is vm-page-size * vm-pages
#
# With the default of 256-bytes memory pages and 104857600 pages Redis will
# use a 25 GB swap file, that will use rougly 13 MB of RAM for the page table.
vm
-
pages
104857600
############################### ADVANCED CONFIG ###############################
...
...
staticsymbols.h
View file @
4ef8de8a
...
...
@@ -17,6 +17,7 @@ static struct redisFunctionSym symsTable[] = {
{
"call"
,(
unsigned
long
)
call
},
{
"closeTimedoutClients"
,(
unsigned
long
)
closeTimedoutClients
},
{
"compareStringObjects"
,(
unsigned
long
)
compareStringObjects
},
{
"computeObjectSwappability"
,(
unsigned
long
)
computeObjectSwappability
},
{
"createClient"
,(
unsigned
long
)
createClient
},
{
"createListObject"
,(
unsigned
long
)
createListObject
},
{
"createObject"
,(
unsigned
long
)
createObject
},
...
...
@@ -40,6 +41,7 @@ static struct redisFunctionSym symsTable[] = {
{
"dictRedisObjectDestructor"
,(
unsigned
long
)
dictRedisObjectDestructor
},
{
"dictVanillaFree"
,(
unsigned
long
)
dictVanillaFree
},
{
"dupClientReplyValue"
,(
unsigned
long
)
dupClientReplyValue
},
{
"dupStringObject"
,(
unsigned
long
)
dupStringObject
},
{
"echoCommand"
,(
unsigned
long
)
echoCommand
},
{
"execCommand"
,(
unsigned
long
)
execCommand
},
{
"existsCommand"
,(
unsigned
long
)
existsCommand
},
...
...
@@ -119,6 +121,7 @@ static struct redisFunctionSym symsTable[] = {
{
"rdbLoadIntegerObject"
,(
unsigned
long
)
rdbLoadIntegerObject
},
{
"rdbLoadLen"
,(
unsigned
long
)
rdbLoadLen
},
{
"rdbLoadLzfStringObject"
,(
unsigned
long
)
rdbLoadLzfStringObject
},
{
"rdbLoadObject"
,(
unsigned
long
)
rdbLoadObject
},
{
"rdbLoadStringObject"
,(
unsigned
long
)
rdbLoadStringObject
},
{
"rdbLoadTime"
,(
unsigned
long
)
rdbLoadTime
},
{
"rdbLoadType"
,(
unsigned
long
)
rdbLoadType
},
...
...
@@ -128,10 +131,13 @@ static struct redisFunctionSym symsTable[] = {
{
"rdbSaveDoubleValue"
,(
unsigned
long
)
rdbSaveDoubleValue
},
{
"rdbSaveLen"
,(
unsigned
long
)
rdbSaveLen
},
{
"rdbSaveLzfStringObject"
,(
unsigned
long
)
rdbSaveLzfStringObject
},
{
"rdbSaveObject"
,(
unsigned
long
)
rdbSaveObject
},
{
"rdbSaveStringObject"
,(
unsigned
long
)
rdbSaveStringObject
},
{
"rdbSaveStringObjectRaw"
,(
unsigned
long
)
rdbSaveStringObjectRaw
},
{
"rdbSaveTime"
,(
unsigned
long
)
rdbSaveTime
},
{
"rdbSaveType"
,(
unsigned
long
)
rdbSaveType
},
{
"rdbSavedObjectLen"
,(
unsigned
long
)
rdbSavedObjectLen
},
{
"rdbSavedObjectPages"
,(
unsigned
long
)
rdbSavedObjectPages
},
{
"rdbTryIntegerEncoding"
,(
unsigned
long
)
rdbTryIntegerEncoding
},
{
"readQueryFromClient"
,(
unsigned
long
)
readQueryFromClient
},
{
"redisLog"
,(
unsigned
long
)
redisLog
},
...
...
@@ -193,6 +199,16 @@ static struct redisFunctionSym symsTable[] = {
{
"typeCommand"
,(
unsigned
long
)
typeCommand
},
{
"unblockClient"
,(
unsigned
long
)
unblockClient
},
{
"updateSlavesWaitingBgsave"
,(
unsigned
long
)
updateSlavesWaitingBgsave
},
{
"vmFindContiguousPages"
,(
unsigned
long
)
vmFindContiguousPages
},
{
"vmFreePage"
,(
unsigned
long
)
vmFreePage
},
{
"vmInit"
,(
unsigned
long
)
vmInit
},
{
"vmLoadObject"
,(
unsigned
long
)
vmLoadObject
},
{
"vmMarkPageFree"
,(
unsigned
long
)
vmMarkPageFree
},
{
"vmMarkPageUsed"
,(
unsigned
long
)
vmMarkPageUsed
},
{
"vmMarkPagesFree"
,(
unsigned
long
)
vmMarkPagesFree
},
{
"vmMarkPagesUsed"
,(
unsigned
long
)
vmMarkPagesUsed
},
{
"vmSwapObject"
,(
unsigned
long
)
vmSwapObject
},
{
"vmSwapOneObject"
,(
unsigned
long
)
vmSwapOneObject
},
{
"yesnotoi"
,(
unsigned
long
)
yesnotoi
},
{
"zaddCommand"
,(
unsigned
long
)
zaddCommand
},
{
"zaddGenericCommand"
,(
unsigned
long
)
zaddGenericCommand
},
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment