Commit 86d48efb authored by antirez's avatar antirez
Browse files

Lazyfree: Convert Sets to use plains SDS (several commits squashed).

parent 4ff3c17a
...@@ -415,8 +415,8 @@ void scanCallback(void *privdata, const dictEntry *de) { ...@@ -415,8 +415,8 @@ void scanCallback(void *privdata, const dictEntry *de) {
sds sdskey = dictGetKey(de); sds sdskey = dictGetKey(de);
key = createStringObject(sdskey, sdslen(sdskey)); key = createStringObject(sdskey, sdslen(sdskey));
} else if (o->type == OBJ_SET) { } else if (o->type == OBJ_SET) {
key = dictGetKey(de); sds keysds = dictGetKey(de);
incrRefCount(key); key = createStringObject(keysds,sdslen(keysds));
} else if (o->type == OBJ_HASH) { } else if (o->type == OBJ_HASH) {
key = dictGetKey(de); key = dictGetKey(de);
incrRefCount(key); incrRefCount(key);
......
...@@ -164,7 +164,9 @@ void computeDatasetDigest(unsigned char *final) { ...@@ -164,7 +164,9 @@ void computeDatasetDigest(unsigned char *final) {
} else if (o->type == OBJ_SET) { } else if (o->type == OBJ_SET) {
setTypeIterator *si = setTypeInitIterator(o); setTypeIterator *si = setTypeInitIterator(o);
robj *ele; robj *ele;
while((ele = setTypeNextObject(si)) != NULL) { sds sdsele;
while((sdsele = setTypeNextObject(si)) != NULL) {
ele = createObject(OBJ_STRING,sdsele);
xorObjectDigest(digest,ele); xorObjectDigest(digest,ele);
decrRefCount(ele); decrRefCount(ele);
} }
......
...@@ -117,13 +117,13 @@ client *createClient(int fd) { ...@@ -117,13 +117,13 @@ client *createClient(int fd) {
listSetDupMethod(c->reply,dupClientReplyValue); listSetDupMethod(c->reply,dupClientReplyValue);
c->btype = BLOCKED_NONE; c->btype = BLOCKED_NONE;
c->bpop.timeout = 0; c->bpop.timeout = 0;
c->bpop.keys = dictCreate(&setDictType,NULL); c->bpop.keys = dictCreate(&objectKeyPointerValueDictType,NULL);
c->bpop.target = NULL; c->bpop.target = NULL;
c->bpop.numreplicas = 0; c->bpop.numreplicas = 0;
c->bpop.reploffset = 0; c->bpop.reploffset = 0;
c->woff = 0; c->woff = 0;
c->watched_keys = listCreate(); c->watched_keys = listCreate();
c->pubsub_channels = dictCreate(&setDictType,NULL); c->pubsub_channels = dictCreate(&objectKeyPointerValueDictType,NULL);
c->pubsub_patterns = listCreate(); c->pubsub_patterns = listCreate();
c->peerid = NULL; c->peerid = NULL;
listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid); listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid);
......
...@@ -364,13 +364,17 @@ int checkType(client *c, robj *o, int type) { ...@@ -364,13 +364,17 @@ int checkType(client *c, robj *o, int type) {
return 0; return 0;
} }
int isSdsRepresentableAsLongLong(sds s, long long *llval) {
return string2ll(s,sdslen(s),llval) ? C_OK : C_ERR;
}
int isObjectRepresentableAsLongLong(robj *o, long long *llval) { int isObjectRepresentableAsLongLong(robj *o, long long *llval) {
serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);
if (o->encoding == OBJ_ENCODING_INT) { if (o->encoding == OBJ_ENCODING_INT) {
if (llval) *llval = (long) o->ptr; if (llval) *llval = (long) o->ptr;
return C_OK; return C_OK;
} else { } else {
return string2ll(o->ptr,sdslen(o->ptr),llval) ? C_OK : C_ERR; return isSdsRepresentableAsLongLong(o->ptr,llval);
} }
} }
......
...@@ -43,6 +43,7 @@ ...@@ -43,6 +43,7 @@
#define RDB_LOAD_NONE 0 #define RDB_LOAD_NONE 0
#define RDB_LOAD_ENC (1<<0) #define RDB_LOAD_ENC (1<<0)
#define RDB_LOAD_PLAIN (1<<1) #define RDB_LOAD_PLAIN (1<<1)
#define RDB_LOAD_SDS (1<<2)
#define rdbExitReportCorruptRDB(reason) rdbCheckThenExit(reason, __LINE__); #define rdbExitReportCorruptRDB(reason) rdbCheckThenExit(reason, __LINE__);
...@@ -179,6 +180,7 @@ int rdbEncodeInteger(long long value, unsigned char *enc) { ...@@ -179,6 +180,7 @@ int rdbEncodeInteger(long long value, unsigned char *enc) {
* rdbGenerincLoadStringObject() for more info. */ * rdbGenerincLoadStringObject() for more info. */
void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags) { void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags) {
int plain = flags & RDB_LOAD_PLAIN; int plain = flags & RDB_LOAD_PLAIN;
int sds = flags & RDB_LOAD_SDS;
int encode = flags & RDB_LOAD_ENC; int encode = flags & RDB_LOAD_ENC;
unsigned char enc[4]; unsigned char enc[4];
long long val; long long val;
...@@ -200,10 +202,10 @@ void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags) { ...@@ -200,10 +202,10 @@ void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags) {
val = 0; /* anti-warning */ val = 0; /* anti-warning */
rdbExitReportCorruptRDB("Unknown RDB integer encoding type"); rdbExitReportCorruptRDB("Unknown RDB integer encoding type");
} }
if (plain) { if (plain || sds) {
char buf[LONG_STR_SIZE], *p; char buf[LONG_STR_SIZE], *p;
int len = ll2string(buf,sizeof(buf),val); int len = ll2string(buf,sizeof(buf),val);
p = zmalloc(len); p = plain ? zmalloc(len) : sdsnewlen(NULL,len);
memcpy(p,buf,len); memcpy(p,buf,len);
return p; return p;
} else if (encode) { } else if (encode) {
...@@ -280,9 +282,10 @@ ssize_t rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) { ...@@ -280,9 +282,10 @@ ssize_t rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) {
* rdbGenericLoadStringObject() function. */ * rdbGenericLoadStringObject() function. */
void *rdbLoadLzfStringObject(rio *rdb, int flags) { void *rdbLoadLzfStringObject(rio *rdb, int flags) {
int plain = flags & RDB_LOAD_PLAIN; int plain = flags & RDB_LOAD_PLAIN;
int sds = flags & RDB_LOAD_SDS;
unsigned int len, clen; unsigned int len, clen;
unsigned char *c = NULL; unsigned char *c = NULL;
sds val = NULL; char *val = NULL;
if ((clen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; if ((clen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
...@@ -292,7 +295,7 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags) { ...@@ -292,7 +295,7 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags) {
if (plain) { if (plain) {
val = zmalloc(len); val = zmalloc(len);
} else { } else {
if ((val = sdsnewlen(NULL,len)) == NULL) goto err; val = sdsnewlen(NULL,len);
} }
/* Load the compressed representation and uncompress it to target. */ /* Load the compressed representation and uncompress it to target. */
...@@ -300,10 +303,11 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags) { ...@@ -300,10 +303,11 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags) {
if (lzf_decompress(c,clen,val,len) == 0) goto err; if (lzf_decompress(c,clen,val,len) == 0) goto err;
zfree(c); zfree(c);
if (plain) if (plain || sds) {
return val; return val;
else } else {
return createObject(OBJ_STRING,val); return createObject(OBJ_STRING,val);
}
err: err:
zfree(c); zfree(c);
if (plain) if (plain)
...@@ -366,7 +370,7 @@ ssize_t rdbSaveLongLongAsStringObject(rio *rdb, long long value) { ...@@ -366,7 +370,7 @@ ssize_t rdbSaveLongLongAsStringObject(rio *rdb, long long value) {
return nwritten; return nwritten;
} }
/* Like rdbSaveStringObjectRaw() but handle encoded objects */ /* Like rdbSaveRawString() gets a Redis object instead. */
int rdbSaveStringObject(rio *rdb, robj *obj) { int rdbSaveStringObject(rio *rdb, robj *obj) {
/* Avoid to decode the object, then encode it again, if the /* Avoid to decode the object, then encode it again, if the
* object is already integer encoded. */ * object is already integer encoded. */
...@@ -387,10 +391,12 @@ int rdbSaveStringObject(rio *rdb, robj *obj) { ...@@ -387,10 +391,12 @@ int rdbSaveStringObject(rio *rdb, robj *obj) {
* no longer guarantees that obj->ptr is an SDS string. * no longer guarantees that obj->ptr is an SDS string.
* RDB_LOAD_PLAIN: Return a plain string allocated with zmalloc() * RDB_LOAD_PLAIN: Return a plain string allocated with zmalloc()
* instead of a Redis object. * instead of a Redis object.
* RDB_LOAD_SDS: Return an SDS string instead of a Redis object.
*/ */
void *rdbGenericLoadStringObject(rio *rdb, int flags) { void *rdbGenericLoadStringObject(rio *rdb, int flags) {
int encode = flags & RDB_LOAD_ENC; int encode = flags & RDB_LOAD_ENC;
int plain = flags & RDB_LOAD_PLAIN; int plain = flags & RDB_LOAD_PLAIN;
int sds = flags & RDB_LOAD_SDS;
int isencoded; int isencoded;
uint32_t len; uint32_t len;
...@@ -409,7 +415,17 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags) { ...@@ -409,7 +415,17 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags) {
} }
if (len == RDB_LENERR) return NULL; if (len == RDB_LENERR) return NULL;
if (!plain) { if (plain || sds) {
void *buf = plain ? zmalloc(len) : sdsnewlen(NULL,len);
if (len && rioRead(rdb,buf,len) == 0) {
if (plain)
zfree(buf);
else
sdsfree(buf);
return NULL;
}
return buf;
} else {
robj *o = encode ? createStringObject(NULL,len) : robj *o = encode ? createStringObject(NULL,len) :
createRawStringObject(NULL,len); createRawStringObject(NULL,len);
if (len && rioRead(rdb,o->ptr,len) == 0) { if (len && rioRead(rdb,o->ptr,len) == 0) {
...@@ -417,13 +433,6 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags) { ...@@ -417,13 +433,6 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags) {
return NULL; return NULL;
} }
return o; return o;
} else {
void *buf = zmalloc(len);
if (len && rioRead(rdb,buf,len) == 0) {
zfree(buf);
return NULL;
}
return buf;
} }
} }
...@@ -583,8 +592,9 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) { ...@@ -583,8 +592,9 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) {
nwritten += n; nwritten += n;
while((de = dictNext(di)) != NULL) { while((de = dictNext(di)) != NULL) {
robj *eleobj = dictGetKey(de); sds ele = dictGetKey(de);
if ((n = rdbSaveStringObject(rdb,eleobj)) == -1) return -1; if ((n = rdbSaveRawString(rdb,(unsigned char*)ele,sdslen(ele)))
== -1) return -1;
nwritten += n; nwritten += n;
} }
dictReleaseIterator(di); dictReleaseIterator(di);
...@@ -959,7 +969,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -959,7 +969,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
decrRefCount(ele); decrRefCount(ele);
} }
} else if (rdbtype == RDB_TYPE_SET) { } else if (rdbtype == RDB_TYPE_SET) {
/* Read list/set value */ /* Read Set value */
if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
/* Use a regular set when there are too many entries. */ /* Use a regular set when there are too many entries. */
...@@ -973,15 +983,17 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -973,15 +983,17 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
o = createIntsetObject(); o = createIntsetObject();
} }
/* Load every single element of the list/set */ /* Load every single element of the set */
for (i = 0; i < len; i++) { for (i = 0; i < len; i++) {
long long llval; long long llval;
if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL; sds sdsele;
ele = tryObjectEncoding(ele);
if ((sdsele = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS)) == NULL)
return NULL;
if (o->encoding == OBJ_ENCODING_INTSET) { if (o->encoding == OBJ_ENCODING_INTSET) {
/* Fetch integer value from element */ /* Fetch integer value from element */
if (isObjectRepresentableAsLongLong(ele,&llval) == C_OK) { if (isSdsRepresentableAsLongLong(sdsele,&llval) == C_OK) {
o->ptr = intsetAdd(o->ptr,llval,NULL); o->ptr = intsetAdd(o->ptr,llval,NULL);
} else { } else {
setTypeConvert(o,OBJ_ENCODING_HT); setTypeConvert(o,OBJ_ENCODING_HT);
...@@ -992,9 +1004,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -992,9 +1004,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* This will also be called when the set was just converted /* This will also be called when the set was just converted
* to a regular hash table encoded set */ * to a regular hash table encoded set */
if (o->encoding == OBJ_ENCODING_HT) { if (o->encoding == OBJ_ENCODING_HT) {
dictAdd((dict*)o->ptr,ele,NULL); dictAdd((dict*)o->ptr,sdsele,NULL);
} else { } else {
decrRefCount(ele); sdsfree(sdsele);
} }
} }
} else if (rdbtype == RDB_TYPE_ZSET) { } else if (rdbtype == RDB_TYPE_ZSET) {
......
...@@ -531,8 +531,9 @@ unsigned int dictEncObjHash(const void *key) { ...@@ -531,8 +531,9 @@ unsigned int dictEncObjHash(const void *key) {
} }
} }
/* Sets type hash table */ /* Generic hash table type where keys are Redis Objects, Values
dictType setDictType = { * dummy pointers. */
dictType objectKeyPointerValueDictType = {
dictEncObjHash, /* hash function */ dictEncObjHash, /* hash function */
NULL, /* key dup */ NULL, /* key dup */
NULL, /* val dup */ NULL, /* val dup */
...@@ -541,6 +542,16 @@ dictType setDictType = { ...@@ -541,6 +542,16 @@ dictType setDictType = {
NULL /* val destructor */ NULL /* val destructor */
}; };
/* Set dictionary type. Keys are SDS strings, values are ot used. */
dictType setDictType = {
dictSdsHash, /* hash function */
NULL, /* key dup */
NULL, /* val dup */
dictSdsKeyCompare, /* key compare */
dictSdsDestructor, /* key destructor */
NULL /* val destructor */
};
/* Sorted sets hash (note: a skiplist is used in addition to the hash table) */ /* Sorted sets hash (note: a skiplist is used in addition to the hash table) */
dictType zsetDictType = { dictType zsetDictType = {
dictEncObjHash, /* hash function */ dictEncObjHash, /* hash function */
...@@ -1834,7 +1845,7 @@ void initServer(void) { ...@@ -1834,7 +1845,7 @@ void initServer(void) {
server.db[j].dict = dictCreate(&dbDictType,NULL); server.db[j].dict = dictCreate(&dbDictType,NULL);
server.db[j].expires = dictCreate(&keyptrDictType,NULL); server.db[j].expires = dictCreate(&keyptrDictType,NULL);
server.db[j].blocking_keys = dictCreate(&keylistDictType,NULL); server.db[j].blocking_keys = dictCreate(&keylistDictType,NULL);
server.db[j].ready_keys = dictCreate(&setDictType,NULL); server.db[j].ready_keys = dictCreate(&objectKeyPointerValueDictType,NULL);
server.db[j].watched_keys = dictCreate(&keylistDictType,NULL); server.db[j].watched_keys = dictCreate(&keylistDictType,NULL);
server.db[j].eviction_pool = evictionPoolAlloc(); server.db[j].eviction_pool = evictionPoolAlloc();
server.db[j].id = j; server.db[j].id = j;
......
...@@ -1043,6 +1043,7 @@ typedef struct { ...@@ -1043,6 +1043,7 @@ typedef struct {
extern struct redisServer server; extern struct redisServer server;
extern struct sharedObjectsStruct shared; extern struct sharedObjectsStruct shared;
extern dictType objectKeyPointerValueDictType;
extern dictType setDictType; extern dictType setDictType;
extern dictType zsetDictType; extern dictType zsetDictType;
extern dictType clusterNodesDictType; extern dictType clusterNodesDictType;
...@@ -1173,6 +1174,7 @@ robj *createStringObject(const char *ptr, size_t len); ...@@ -1173,6 +1174,7 @@ robj *createStringObject(const char *ptr, size_t len);
robj *createRawStringObject(const char *ptr, size_t len); robj *createRawStringObject(const char *ptr, size_t len);
robj *createEmbeddedStringObject(const char *ptr, size_t len); robj *createEmbeddedStringObject(const char *ptr, size_t len);
robj *dupStringObject(robj *o); robj *dupStringObject(robj *o);
int isSdsRepresentableAsLongLong(sds s, long long *llval);
int isObjectRepresentableAsLongLong(robj *o, long long *llongval); int isObjectRepresentableAsLongLong(robj *o, long long *llongval);
robj *tryObjectEncoding(robj *o); robj *tryObjectEncoding(robj *o);
robj *getDecodedObject(robj *o); robj *getDecodedObject(robj *o);
...@@ -1313,15 +1315,15 @@ unsigned int getLRUClock(void); ...@@ -1313,15 +1315,15 @@ unsigned int getLRUClock(void);
const char *maxmemoryToString(void); const char *maxmemoryToString(void);
/* Set data type */ /* Set data type */
robj *setTypeCreate(robj *value); robj *setTypeCreate(sds value);
int setTypeAdd(robj *subject, robj *value); int setTypeAdd(robj *subject, sds value);
int setTypeRemove(robj *subject, robj *value); int setTypeRemove(robj *subject, sds value);
int setTypeIsMember(robj *subject, robj *value); int setTypeIsMember(robj *subject, sds value);
setTypeIterator *setTypeInitIterator(robj *subject); setTypeIterator *setTypeInitIterator(robj *subject);
void setTypeReleaseIterator(setTypeIterator *si); void setTypeReleaseIterator(setTypeIterator *si);
int setTypeNext(setTypeIterator *si, robj **objele, int64_t *llele); int setTypeNext(setTypeIterator *si, sds *sdsele, int64_t *llele);
robj *setTypeNextObject(setTypeIterator *si); sds setTypeNextObject(setTypeIterator *si);
int setTypeRandomElement(robj *setobj, robj **objele, int64_t *llele); int setTypeRandomElement(robj *setobj, sds *sdsele, int64_t *llele);
unsigned long setTypeRandomElements(robj *set, unsigned long count, robj *aux_set); unsigned long setTypeRandomElements(robj *set, unsigned long count, robj *aux_set);
unsigned long setTypeSize(robj *subject); unsigned long setTypeSize(robj *subject);
void setTypeConvert(robj *subject, int enc); void setTypeConvert(robj *subject, int enc);
......
...@@ -380,9 +380,9 @@ void sortCommand(client *c) { ...@@ -380,9 +380,9 @@ void sortCommand(client *c) {
listTypeReleaseIterator(li); listTypeReleaseIterator(li);
} else if (sortval->type == OBJ_SET) { } else if (sortval->type == OBJ_SET) {
setTypeIterator *si = setTypeInitIterator(sortval); setTypeIterator *si = setTypeInitIterator(sortval);
robj *ele; sds sdsele;
while((ele = setTypeNextObject(si)) != NULL) { while((sdsele = setTypeNextObject(si)) != NULL) {
vector[j].obj = ele; vector[j].obj = createObject(OBJ_STRING,sdsele);
vector[j].u.score = 0; vector[j].u.score = 0;
vector[j].u.cmpobj = NULL; vector[j].u.cmpobj = NULL;
j++; j++;
......
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment