Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
10c43610
Commit
10c43610
authored
Mar 25, 2009
by
antirez
Browse files
Nasty bug of the new DB format fixed, objects sharing implemented
parent
d9f650b9
Changes
3
Hide whitespace changes
Inline
Side-by-side
TODO
View file @
10c43610
...
...
@@ -8,5 +8,6 @@
- check 'server.dirty' everywere
- replication automated tests
- a command, or an external tool, to perform the MD5SUM of the whole dataset, so that if the dataset between two servers is identical, so will be the MD5SUM
- objects sharing, "objectsharing yes", "objectsharingpool 1024"
* Include Lua and Perl bindings
redis.c
View file @
10c43610
...
...
@@ -95,10 +95,10 @@
* 00|000000 => if the two MSB are 00 the len is the 6 bits of this byte
* 01|000000 00000000 => 01, the len is 14 byes, 6 bits + 8 bits of next byte
* 10|000000 [32 bit integer] => if it's 01, a full 32 bit len will follow
* 11|000000
[64 bit integer] => if it's 11, a full 64 bit len will follow
* 11|000000
reserved for future uses
*
*
64 bit lengths are not used currently. Lenghts up to 63 are stored using
*
a single byte, most DB keys, and may
values, will fit inside. */
*
Lenghts up to 63 are stored using a single byte, most DB keys, and may
* values, will fit inside. */
#define REDIS_RDB_6BITLEN 0
#define REDIS_RDB_14BITLEN 1
#define REDIS_RDB_32BITLEN 2
...
...
@@ -173,6 +173,8 @@ struct redisServer {
int port;
int fd;
dict **dict;
dict *sharingpool;
unsigned int sharingpoolsize;
long long dirty; /* changes to DB from the last save */
list *clients;
list *slaves, *monitors;
...
...
@@ -199,6 +201,7 @@ struct redisServer {
char *logfile;
char *bindaddr;
char *dbfilename;
int shareobjects;
/* Replication related */
int isslave;
char *masterhost;
...
...
@@ -258,6 +261,7 @@ static int rdbSaveBackground(char *filename);
static robj *createStringObject(char *ptr, size_t len);
static void replicationFeedSlaves(list *slaves, struct redisCommand *cmd, int dictid, robj **argv, int argc);
static int syncWithMaster(void);
static robj *tryObjectSharing(robj *o);
static void pingCommand(redisClient *c);
static void echoCommand(redisClient *c);
...
...
@@ -627,7 +631,8 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
redisLog(REDIS_DEBUG,"%d clients connected (%d slaves), %d bytes in use",
listLength(server.clients)-listLength(server.slaves),
listLength(server.slaves),
server
.
usedmemory
);
server.usedmemory,
dictGetHashTableUsed(server.sharingpool));
}
/* Close connections of timedout clients */
...
...
@@ -739,6 +744,7 @@ static void initServerConfig() {
server.daemonize = 0;
server.pidfile = "/var/run/redis.pid";
server.dbfilename = "dump.rdb";
server.shareobjects = 0;
ResetServerSaveParams();
appendServerSaveParams(60*60,1); /* save after 1 hour and 1 change */
...
...
@@ -765,6 +771,8 @@ static void initServer() {
createSharedObjects();
server.el = aeCreateEventLoop();
server.dict = zmalloc(sizeof(dict*)*server.dbnum);
server.sharingpool = dictCreate(&setDictType,NULL);
server.sharingpoolsize = 1024;
if (!server.dict || !server.clients || !server.slaves || !server.monitors || !server.el || !server.objfreelist)
oom("server initialization"); /* Fatal OOM */
server.fd = anetTcpServer(server.neterr, server.port, server.bindaddr);
...
...
@@ -772,11 +780,8 @@ static void initServer() {
redisLog(REDIS_WARNING, "Opening TCP port: %s", server.neterr);
exit(1);
}
for
(
j
=
0
;
j
<
server
.
dbnum
;
j
++
)
{
for (j = 0; j < server.dbnum; j++)
server.dict[j] = dictCreate(&hashDictType,NULL);
if
(
!
server
.
dict
[
j
])
oom
(
"dictCreate"
);
/* Fatal OOM */
}
server.cronloops = 0;
server.bgsaveinprogress = 0;
server.lastsave = time(NULL);
...
...
@@ -895,6 +900,13 @@ static void loadServerConfig(char *filename) {
else {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcmp(argv[0],"shareobjects") && argc == 2) {
sdstolower(argv[1]);
if (!strcmp(argv[1],"yes")) server.shareobjects = 1;
else if (!strcmp(argv[1],"no")) server.shareobjects = 0;
else {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcmp(argv[0],"daemonize") && argc == 2) {
sdstolower(argv[1]);
if (!strcmp(argv[1],"yes")) server.daemonize = 1;
...
...
@@ -1102,6 +1114,12 @@ static int processCommand(redisClient *c) {
return 1;
}
}
/* Let's try to share objects on the command arguments vector */
if (server.shareobjects) {
int j;
for(j = 1; j < c->argc; j++)
c->argv[j] = tryObjectSharing(c->argv[j]);
}
/* Exec the command */
dirty = server.dirty;
cmd->proc(c);
...
...
@@ -1412,6 +1430,51 @@ static void decrRefCount(void *obj) {
}
}
/* Try to share an object against the shared objects pool */
static robj *tryObjectSharing(robj *o) {
struct dictEntry *de;
unsigned long c;
if (server.shareobjects == 0) return o;
assert(o->type == REDIS_STRING);
de = dictFind(server.sharingpool,o);
if (de) {
robj *shared = dictGetEntryKey(de);
c = ((unsigned long) dictGetEntryVal(de))+1;
dictGetEntryVal(de) = (void*) c;
incrRefCount(shared);
decrRefCount(o);
return shared;
} else {
/* Here we are using a stream algorihtm: Every time an object is
* shared we increment its count, everytime there is a miss we
* recrement the counter of a random object. If this object reaches
* zero we remove the object and put the current object instead. */
if (dictGetHashTableUsed(server.sharingpool) >=
server.sharingpoolsize) {
de = dictGetRandomKey(server.sharingpool);
assert(de != NULL);
c = ((unsigned long) dictGetEntryVal(de))-1;
dictGetEntryVal(de) = (void*) c;
if (c == 0) {
dictDelete(server.sharingpool,de->key);
}
} else {
c = 0; /* If the pool is empty we want to add this object */
}
if (c == 0) {
int retval;
retval = dictAdd(server.sharingpool,o,(void*)1);
assert(retval == DICT_OK);
incrRefCount(o);
}
return o;
}
}
/*============================ DB saving/loading ============================ */
static int rdbSaveType(FILE *fp, unsigned char type) {
...
...
@@ -1424,16 +1487,16 @@ static int rdbSaveLen(FILE *fp, uint32_t len) {
if (len < (1<<6)) {
/* Save a 6 bit len */
buf
[
0
]
=
(
len
&
0xFF
)
|
REDIS_RDB_6BITLEN
;
buf[0] = (len&0xFF)|
(
REDIS_RDB_6BITLEN
<<6)
;
if (fwrite(buf,1,1,fp) == 0) return -1;
} else if (len < (1<<14)) {
/* Save a 14 bit len */
buf
[
0
]
=
((
len
>>
8
)
&
0xFF
)
|
REDIS_RDB_14BITLEN
;
buf[0] = ((len>>8)&0xFF)|
(
REDIS_RDB_14BITLEN
<<6)
;
buf[1] = len&0xFF;
if (fwrite(buf,4,1,fp) == 0) return -1;
} else {
/* Save a 32 bit len */
buf
[
0
]
=
REDIS_RDB_32BITLEN
;
buf[0] =
(
REDIS_RDB_32BITLEN
<<6)
;
if (fwrite(buf,1,1,fp) == 0) return -1;
len = htonl(len);
if (fwrite(&len,4,1,fp) == 0) return -1;
...
...
@@ -1441,6 +1504,14 @@ static int rdbSaveLen(FILE *fp, uint32_t len) {
return 0;
}
static int rdbSaveStringObject(FILE *fp, robj *obj) {
size_t len = sdslen(obj->ptr);
if (rdbSaveLen(fp,len) == -1) return -1;
if (len && fwrite(obj->ptr,len,1,fp) == 0) return -1;
return 0;
}
/* Save the DB on disk. Return REDIS_ERR on error, REDIS_OK on success */
static int rdbSave(char *filename) {
dictIterator *di = NULL;
...
...
@@ -1475,15 +1546,10 @@ static int rdbSave(char *filename) {
robj *o = dictGetEntryVal(de);
if (rdbSaveType(fp,o->type) == -1) goto werr;
if
(
rdbSaveLen
(
fp
,
sdslen
(
key
->
ptr
))
==
-
1
)
goto
werr
;
if
(
fwrite
(
key
->
ptr
,
sdslen
(
key
->
ptr
),
1
,
fp
)
==
0
)
goto
werr
;
if (rdbSaveStringObject(fp,key) == -1) goto werr;
if (o->type == REDIS_STRING) {
/* Save a string value */
sds
sval
=
o
->
ptr
;
if
(
rdbSaveLen
(
fp
,
sdslen
(
sval
))
==
-
1
)
goto
werr
;
if
(
sdslen
(
sval
)
&&
fwrite
(
sval
,
sdslen
(
sval
),
1
,
fp
)
==
0
)
goto
werr
;
if (rdbSaveStringObject(fp,o) == -1) goto werr;
} else if (o->type == REDIS_LIST) {
/* Save a list value */
list *list = o->ptr;
...
...
@@ -1493,10 +1559,7 @@ static int rdbSave(char *filename) {
while(ln) {
robj *eleobj = listNodeValue(ln);
if
(
rdbSaveLen
(
fp
,
sdslen
(
eleobj
->
ptr
))
==
-
1
)
goto
werr
;
if
(
sdslen
(
eleobj
->
ptr
)
&&
fwrite
(
eleobj
->
ptr
,
sdslen
(
eleobj
->
ptr
),
1
,
fp
)
==
0
)
goto
werr
;
if (rdbSaveStringObject(fp,eleobj) == -1) goto werr;
ln = ln->next;
}
} else if (o->type == REDIS_SET) {
...
...
@@ -1508,13 +1571,9 @@ static int rdbSave(char *filename) {
if (!set) oom("dictGetIteraotr");
if (rdbSaveLen(fp,dictGetHashTableUsed(set)) == -1) goto werr;
while((de = dictNext(di)) != NULL) {
robj
*
eleobj
;
robj *eleobj
= dictGetEntryKey(de)
;
eleobj
=
dictGetEntryKey
(
de
);
if
(
rdbSaveLen
(
fp
,
sdslen
(
eleobj
->
ptr
))
==
-
1
)
goto
werr
;
if
(
sdslen
(
eleobj
->
ptr
)
&&
fwrite
(
eleobj
->
ptr
,
sdslen
(
eleobj
->
ptr
),
1
,
fp
)
==
0
)
goto
werr
;
if (rdbSaveStringObject(fp,eleobj) == -1) goto werr;
}
dictReleaseIterator(di);
} else {
...
...
@@ -1600,7 +1659,6 @@ static uint32_t rdbLoadLen(FILE *fp, int rdbver) {
return ntohl(len);
}
}
return
0
;
}
static robj *rdbLoadStringObject(FILE*fp,int rdbver) {
...
...
@@ -1613,7 +1671,7 @@ static robj *rdbLoadStringObject(FILE*fp,int rdbver) {
sdsfree(val);
return NULL;
}
return
createObject
(
REDIS_STRING
,
val
);
return
tryObjectSharing(
createObject(REDIS_STRING,val)
)
;
}
static int rdbLoad(char *filename) {
...
...
@@ -1625,7 +1683,6 @@ static int rdbLoad(char *filename) {
dict *d = server.dict[0];
char buf[1024];
int rdbver;
fp = fopen(filename,"r");
if (!fp) return REDIS_ERR;
if (fread(buf,9,1,fp) == 0) goto eoferr;
...
...
redis.conf
View file @
10c43610
...
...
@@ -68,3 +68,9 @@ databases 16
# single TCP packet. Uses a bit more CPU but most of the times it is a win
# in terms of number of queries per second. Use 'yes' if unsure.
glueoutputbuf
yes
# Use object sharing. Can save a lot of memory if you have many common
# string in your dataset, but performs lookups against the shared objects
# pool so it uses more CPU and can be a bit slower. Usually it's a good
# idea.
shareobjects
no
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment