Commit b04ce2a3 authored by Pieter Noordhuis's avatar Pieter Noordhuis
Browse files

Merge master with resolved conflict in src/redis-cli.c

parents 4fe83b55 b4f2e412
...@@ -33,7 +33,6 @@ ...@@ -33,7 +33,6 @@
#include "sds.h" #include "sds.h"
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdarg.h>
#include <string.h> #include <string.h>
#include <ctype.h> #include <ctype.h>
#include "zmalloc.h" #include "zmalloc.h"
...@@ -156,8 +155,8 @@ sds sdscpy(sds s, char *t) { ...@@ -156,8 +155,8 @@ sds sdscpy(sds s, char *t) {
return sdscpylen(s, t, strlen(t)); return sdscpylen(s, t, strlen(t));
} }
sds sdscatprintf(sds s, const char *fmt, ...) { sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
va_list ap; va_list cpy;
char *buf, *t; char *buf, *t;
size_t buflen = 16; size_t buflen = 16;
...@@ -169,9 +168,8 @@ sds sdscatprintf(sds s, const char *fmt, ...) { ...@@ -169,9 +168,8 @@ sds sdscatprintf(sds s, const char *fmt, ...) {
if (buf == NULL) return NULL; if (buf == NULL) return NULL;
#endif #endif
buf[buflen-2] = '\0'; buf[buflen-2] = '\0';
va_start(ap, fmt); va_copy(cpy,ap);
vsnprintf(buf, buflen, fmt, ap); vsnprintf(buf, buflen, fmt, cpy);
va_end(ap);
if (buf[buflen-2] != '\0') { if (buf[buflen-2] != '\0') {
zfree(buf); zfree(buf);
buflen *= 2; buflen *= 2;
...@@ -184,6 +182,15 @@ sds sdscatprintf(sds s, const char *fmt, ...) { ...@@ -184,6 +182,15 @@ sds sdscatprintf(sds s, const char *fmt, ...) {
return t; return t;
} }
sds sdscatprintf(sds s, const char *fmt, ...) {
va_list ap;
char *t;
va_start(ap, fmt);
t = sdscatvprintf(s,fmt,ap);
va_end(ap);
return t;
}
sds sdstrim(sds s, const char *cset) { sds sdstrim(sds s, const char *cset) {
struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr)));
char *start, *end, *sp, *ep; char *start, *end, *sp, *ep;
...@@ -216,13 +223,16 @@ sds sdsrange(sds s, int start, int end) { ...@@ -216,13 +223,16 @@ sds sdsrange(sds s, int start, int end) {
} }
newlen = (start > end) ? 0 : (end-start)+1; newlen = (start > end) ? 0 : (end-start)+1;
if (newlen != 0) { if (newlen != 0) {
if (start >= (signed)len) start = len-1; if (start >= (signed)len) {
if (end >= (signed)len) end = len-1; newlen = 0;
} else if (end >= (signed)len) {
end = len-1;
newlen = (start > end) ? 0 : (end-start)+1; newlen = (start > end) ? 0 : (end-start)+1;
}
} else { } else {
start = 0; start = 0;
} }
if (start != 0) memmove(sh->buf, sh->buf+start, newlen); if (start && newlen) memmove(sh->buf, sh->buf+start, newlen);
sh->buf[newlen] = 0; sh->buf[newlen] = 0;
sh->free = sh->free+(sh->len-newlen); sh->free = sh->free+(sh->len-newlen);
sh->len = newlen; sh->len = newlen;
...@@ -382,3 +392,182 @@ sds sdscatrepr(sds s, char *p, size_t len) { ...@@ -382,3 +392,182 @@ sds sdscatrepr(sds s, char *p, size_t len) {
} }
return sdscatlen(s,"\"",1); return sdscatlen(s,"\"",1);
} }
/* Split a line into arguments, where every argument can be in the
* following programming-language REPL-alike form:
*
* foo bar "newline are supported\n" and "\xff\x00otherstuff"
*
* The number of arguments is stored into *argc, and an array
* of sds is returned. The caller should sdsfree() all the returned
* strings and finally zfree() the array itself.
*
* Note that sdscatrepr() is able to convert back a string into
* a quoted string in the same format sdssplitargs() is able to parse.
*/
sds *sdssplitargs(char *line, int *argc) {
char *p = line;
char *current = NULL;
char **vector = NULL;
*argc = 0;
while(1) {
/* skip blanks */
while(*p && isspace(*p)) p++;
if (*p) {
/* get a token */
int inq=0; /* set to 1 if we are in "quotes" */
int done=0;
if (current == NULL) current = sdsempty();
while(!done) {
if (inq) {
if (*p == '\\' && *(p+1)) {
char c;
p++;
switch(*p) {
case 'n': c = '\n'; break;
case 'r': c = '\r'; break;
case 't': c = '\t'; break;
case 'b': c = '\b'; break;
case 'a': c = '\a'; break;
default: c = *p; break;
}
current = sdscatlen(current,&c,1);
} else if (*p == '"') {
/* closing quote must be followed by a space */
if (*(p+1) && !isspace(*(p+1))) goto err;
done=1;
} else if (!*p) {
/* unterminated quotes */
goto err;
} else {
current = sdscatlen(current,p,1);
}
} else {
switch(*p) {
case ' ':
case '\n':
case '\r':
case '\t':
case '\0':
done=1;
break;
case '"':
inq=1;
break;
default:
current = sdscatlen(current,p,1);
break;
}
}
if (*p) p++;
}
/* add the token to the vector */
vector = zrealloc(vector,((*argc)+1)*sizeof(char*));
vector[*argc] = current;
(*argc)++;
current = NULL;
} else {
return vector;
}
}
err:
while((*argc)--)
sdsfree(vector[*argc]);
zfree(vector);
if (current) sdsfree(current);
return NULL;
}
#ifdef SDS_TEST_MAIN
#include <stdio.h>
#include "testhelp.h"
int main(void) {
{
sds x = sdsnew("foo"), y;
test_cond("Create a string and obtain the length",
sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0)
sdsfree(x);
x = sdsnewlen("foo",2);
test_cond("Create a string with specified length",
sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0)
x = sdscat(x,"bar");
test_cond("Strings concatenation",
sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0);
x = sdscpy(x,"a");
test_cond("sdscpy() against an originally longer string",
sdslen(x) == 1 && memcmp(x,"a\0",2) == 0)
x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk");
test_cond("sdscpy() against an originally shorter string",
sdslen(x) == 33 &&
memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0)
sdsfree(x);
x = sdscatprintf(sdsempty(),"%d",123);
test_cond("sdscatprintf() seems working in the base case",
sdslen(x) == 3 && memcmp(x,"123\0",4) ==0)
sdsfree(x);
x = sdstrim(sdsnew("xxciaoyyy"),"xy");
test_cond("sdstrim() correctly trims characters",
sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0)
y = sdsrange(sdsdup(x),1,1);
test_cond("sdsrange(...,1,1)",
sdslen(y) == 1 && memcmp(y,"i\0",2) == 0)
sdsfree(y);
y = sdsrange(sdsdup(x),1,-1);
test_cond("sdsrange(...,1,-1)",
sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
sdsfree(y);
y = sdsrange(sdsdup(x),-2,-1);
test_cond("sdsrange(...,-2,-1)",
sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0)
sdsfree(y);
y = sdsrange(sdsdup(x),2,1);
test_cond("sdsrange(...,2,1)",
sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
sdsfree(y);
y = sdsrange(sdsdup(x),1,100);
test_cond("sdsrange(...,1,100)",
sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
sdsfree(y);
y = sdsrange(sdsdup(x),100,100);
test_cond("sdsrange(...,100,100)",
sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
sdsfree(y);
sdsfree(x);
x = sdsnew("foo");
y = sdsnew("foa");
test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0)
sdsfree(y);
sdsfree(x);
x = sdsnew("bar");
y = sdsnew("bar");
test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0)
sdsfree(y);
sdsfree(x);
x = sdsnew("aar");
y = sdsnew("bar");
test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0)
}
test_report()
}
#endif
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
#define __SDS_H #define __SDS_H
#include <sys/types.h> #include <sys/types.h>
#include <stdarg.h>
typedef char *sds; typedef char *sds;
...@@ -53,6 +54,7 @@ sds sdscat(sds s, char *t); ...@@ -53,6 +54,7 @@ sds sdscat(sds s, char *t);
sds sdscpylen(sds s, char *t, size_t len); sds sdscpylen(sds s, char *t, size_t len);
sds sdscpy(sds s, char *t); sds sdscpy(sds s, char *t);
sds sdscatvprintf(sds s, const char *fmt, va_list ap);
#ifdef __GNUC__ #ifdef __GNUC__
sds sdscatprintf(sds s, const char *fmt, ...) sds sdscatprintf(sds s, const char *fmt, ...)
__attribute__((format(printf, 2, 3))); __attribute__((format(printf, 2, 3)));
...@@ -70,5 +72,6 @@ void sdstolower(sds s); ...@@ -70,5 +72,6 @@ void sdstolower(sds s);
void sdstoupper(sds s); void sdstoupper(sds s);
sds sdsfromlonglong(long long value); sds sdsfromlonglong(long long value);
sds sdscatrepr(sds s, char *p, size_t len); sds sdscatrepr(sds s, char *p, size_t len);
sds *sdssplitargs(char *line, int *argc);
#endif #endif
/* Solaris specific fixes */ /* Solaris specific fixes */
#if defined(__GNUC__) #if defined(__GNUC__)
#include <math.h>
#undef isnan #undef isnan
#define isnan(x) \ #define isnan(x) \
__extension__({ __typeof (x) __x_a = (x); \ __extension__({ __typeof (x) __x_a = (x); \
......
...@@ -202,7 +202,7 @@ void sortCommand(redisClient *c) { ...@@ -202,7 +202,7 @@ void sortCommand(redisClient *c) {
/* Load the sorting vector with all the objects to sort */ /* Load the sorting vector with all the objects to sort */
switch(sortval->type) { switch(sortval->type) {
case REDIS_LIST: vectorlen = listTypeLength(sortval); break; case REDIS_LIST: vectorlen = listTypeLength(sortval); break;
case REDIS_SET: vectorlen = dictSize((dict*)sortval->ptr); break; case REDIS_SET: vectorlen = setTypeSize(sortval); break;
case REDIS_ZSET: vectorlen = dictSize(((zset*)sortval->ptr)->dict); break; case REDIS_ZSET: vectorlen = dictSize(((zset*)sortval->ptr)->dict); break;
default: vectorlen = 0; redisPanic("Bad SORT type"); /* Avoid GCC warning */ default: vectorlen = 0; redisPanic("Bad SORT type"); /* Avoid GCC warning */
} }
...@@ -219,18 +219,20 @@ void sortCommand(redisClient *c) { ...@@ -219,18 +219,20 @@ void sortCommand(redisClient *c) {
j++; j++;
} }
listTypeReleaseIterator(li); listTypeReleaseIterator(li);
} else { } else if (sortval->type == REDIS_SET) {
dict *set; setTypeIterator *si = setTypeInitIterator(sortval);
robj *ele;
while((ele = setTypeNext(si)) != NULL) {
vector[j].obj = ele;
vector[j].u.score = 0;
vector[j].u.cmpobj = NULL;
j++;
}
setTypeReleaseIterator(si);
} else if (sortval->type == REDIS_ZSET) {
dict *set = ((zset*)sortval->ptr)->dict;
dictIterator *di; dictIterator *di;
dictEntry *setele; dictEntry *setele;
if (sortval->type == REDIS_SET) {
set = sortval->ptr;
} else {
zset *zs = sortval->ptr;
set = zs->dict;
}
di = dictGetIterator(set); di = dictGetIterator(set);
while((setele = dictNext(di)) != NULL) { while((setele = dictNext(di)) != NULL) {
vector[j].obj = dictGetEntryKey(setele); vector[j].obj = dictGetEntryKey(setele);
...@@ -239,6 +241,8 @@ void sortCommand(redisClient *c) { ...@@ -239,6 +241,8 @@ void sortCommand(redisClient *c) {
j++; j++;
} }
dictReleaseIterator(di); dictReleaseIterator(di);
} else {
redisPanic("Unknown type");
} }
redisAssert(j == vectorlen); redisAssert(j == vectorlen);
...@@ -303,7 +307,7 @@ void sortCommand(redisClient *c) { ...@@ -303,7 +307,7 @@ void sortCommand(redisClient *c) {
outputlen = getop ? getop*(end-start+1) : end-start+1; outputlen = getop ? getop*(end-start+1) : end-start+1;
if (storekey == NULL) { if (storekey == NULL) {
/* STORE option not specified, sent the sorting result to client */ /* STORE option not specified, sent the sorting result to client */
addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",outputlen)); addReplyMultiBulkLen(c,outputlen);
for (j = start; j <= end; j++) { for (j = start; j <= end; j++) {
listNode *ln; listNode *ln;
listIter li; listIter li;
...@@ -365,11 +369,11 @@ void sortCommand(redisClient *c) { ...@@ -365,11 +369,11 @@ void sortCommand(redisClient *c) {
* replaced. */ * replaced. */
server.dirty += 1+outputlen; server.dirty += 1+outputlen;
touchWatchedKey(c->db,storekey); touchWatchedKey(c->db,storekey);
addReplySds(c,sdscatprintf(sdsempty(),":%d\r\n",outputlen)); addReplyLongLong(c,outputlen);
} }
/* Cleanup */ /* Cleanup */
if (sortval->type == REDIS_LIST) if (sortval->type == REDIS_LIST || sortval->type == REDIS_SET)
for (j = 0; j < vectorlen; j++) for (j = 0; j < vectorlen; j++)
decrRefCount(vector[j].obj); decrRefCount(vector[j].obj);
decrRefCount(sortval); decrRefCount(sortval);
......
...@@ -249,7 +249,7 @@ void hmsetCommand(redisClient *c) { ...@@ -249,7 +249,7 @@ void hmsetCommand(redisClient *c) {
robj *o; robj *o;
if ((c->argc % 2) == 1) { if ((c->argc % 2) == 1) {
addReplySds(c,sdsnew("-ERR wrong number of arguments for HMSET\r\n")); addReplyError(c,"wrong number of arguments for HMSET");
return; return;
} }
...@@ -315,7 +315,7 @@ void hmgetCommand(redisClient *c) { ...@@ -315,7 +315,7 @@ void hmgetCommand(redisClient *c) {
/* Note the check for o != NULL happens inside the loop. This is /* Note the check for o != NULL happens inside the loop. This is
* done because objects that cannot be found are considered to be * done because objects that cannot be found are considered to be
* an empty hash. The reply should then be a series of NULLs. */ * an empty hash. The reply should then be a series of NULLs. */
addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",c->argc-2)); addReplyMultiBulkLen(c,c->argc-2);
for (i = 2; i < c->argc; i++) { for (i = 2; i < c->argc; i++) {
if (o != NULL && (value = hashTypeGet(o,c->argv[i])) != NULL) { if (o != NULL && (value = hashTypeGet(o,c->argv[i])) != NULL) {
addReplyBulk(c,value); addReplyBulk(c,value);
...@@ -346,21 +346,19 @@ void hlenCommand(redisClient *c) { ...@@ -346,21 +346,19 @@ void hlenCommand(redisClient *c) {
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,o,REDIS_HASH)) return; checkType(c,o,REDIS_HASH)) return;
addReplyUlong(c,hashTypeLength(o)); addReplyLongLong(c,hashTypeLength(o));
} }
void genericHgetallCommand(redisClient *c, int flags) { void genericHgetallCommand(redisClient *c, int flags) {
robj *o, *lenobj, *obj; robj *o, *obj;
unsigned long count = 0; unsigned long count = 0;
hashTypeIterator *hi; hashTypeIterator *hi;
void *replylen = NULL;
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) == NULL if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) == NULL
|| checkType(c,o,REDIS_HASH)) return; || checkType(c,o,REDIS_HASH)) return;
lenobj = createObject(REDIS_STRING,NULL); replylen = addDeferredMultiBulkLength(c);
addReply(c,lenobj);
decrRefCount(lenobj);
hi = hashTypeInitIterator(o); hi = hashTypeInitIterator(o);
while (hashTypeNext(hi) != REDIS_ERR) { while (hashTypeNext(hi) != REDIS_ERR) {
if (flags & REDIS_HASH_KEY) { if (flags & REDIS_HASH_KEY) {
...@@ -377,8 +375,7 @@ void genericHgetallCommand(redisClient *c, int flags) { ...@@ -377,8 +375,7 @@ void genericHgetallCommand(redisClient *c, int flags) {
} }
} }
hashTypeReleaseIterator(hi); hashTypeReleaseIterator(hi);
setDeferredMultiBulkLength(c,replylen,count);
lenobj->ptr = sdscatprintf(sdsempty(),"*%lu\r\n",count);
} }
void hkeysCommand(redisClient *c) { void hkeysCommand(redisClient *c) {
......
...@@ -342,7 +342,7 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) { ...@@ -342,7 +342,7 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) {
server.dirty++; server.dirty++;
} }
addReplyUlong(c,listTypeLength(subject)); addReplyLongLong(c,listTypeLength(subject));
} }
void lpushxCommand(redisClient *c) { void lpushxCommand(redisClient *c) {
...@@ -366,7 +366,7 @@ void linsertCommand(redisClient *c) { ...@@ -366,7 +366,7 @@ void linsertCommand(redisClient *c) {
void llenCommand(redisClient *c) { void llenCommand(redisClient *c) {
robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.czero); robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.czero);
if (o == NULL || checkType(c,o,REDIS_LIST)) return; if (o == NULL || checkType(c,o,REDIS_LIST)) return;
addReplyUlong(c,listTypeLength(o)); addReplyLongLong(c,listTypeLength(o));
} }
void lindexCommand(redisClient *c) { void lindexCommand(redisClient *c) {
...@@ -494,7 +494,7 @@ void lrangeCommand(redisClient *c) { ...@@ -494,7 +494,7 @@ void lrangeCommand(redisClient *c) {
rangelen = (end-start)+1; rangelen = (end-start)+1;
/* Return the result in form of a multi-bulk reply */ /* Return the result in form of a multi-bulk reply */
addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",rangelen)); addReplyMultiBulkLen(c,rangelen);
listTypeIterator *li = listTypeInitIterator(o,start,REDIS_TAIL); listTypeIterator *li = listTypeInitIterator(o,start,REDIS_TAIL);
for (j = 0; j < rangelen; j++) { for (j = 0; j < rangelen; j++) {
redisAssert(listTypeNext(li,&entry)); redisAssert(listTypeNext(li,&entry));
...@@ -594,7 +594,7 @@ void lremCommand(redisClient *c) { ...@@ -594,7 +594,7 @@ void lremCommand(redisClient *c) {
decrRefCount(obj); decrRefCount(obj);
if (listTypeLength(subject) == 0) dbDelete(c->db,c->argv[1]); if (listTypeLength(subject) == 0) dbDelete(c->db,c->argv[1]);
addReplySds(c,sdscatprintf(sdsempty(),":%d\r\n",removed)); addReplyLongLong(c,removed);
if (removed) touchWatchedKey(c->db,c->argv[1]); if (removed) touchWatchedKey(c->db,c->argv[1]);
} }
...@@ -772,7 +772,7 @@ int handleClientsWaitingListPush(redisClient *c, robj *key, robj *ele) { ...@@ -772,7 +772,7 @@ int handleClientsWaitingListPush(redisClient *c, robj *key, robj *ele) {
redisAssert(ln != NULL); redisAssert(ln != NULL);
receiver = ln->value; receiver = ln->value;
addReplySds(receiver,sdsnew("*2\r\n")); addReplyMultiBulkLen(receiver,2);
addReplyBulk(receiver,key); addReplyBulk(receiver,key);
addReplyBulk(receiver,ele); addReplyBulk(receiver,ele);
unblockClientWaitingData(receiver); unblockClientWaitingData(receiver);
...@@ -782,9 +782,20 @@ int handleClientsWaitingListPush(redisClient *c, robj *key, robj *ele) { ...@@ -782,9 +782,20 @@ int handleClientsWaitingListPush(redisClient *c, robj *key, robj *ele) {
/* Blocking RPOP/LPOP */ /* Blocking RPOP/LPOP */
void blockingPopGenericCommand(redisClient *c, int where) { void blockingPopGenericCommand(redisClient *c, int where) {
robj *o; robj *o;
long long lltimeout;
time_t timeout; time_t timeout;
int j; int j;
/* Make sure timeout is an integer value */
if (getLongLongFromObjectOrReply(c,c->argv[c->argc-1],&lltimeout,
"timeout is not an integer") != REDIS_OK) return;
/* Make sure the timeout is not negative */
if (lltimeout < 0) {
addReplyError(c,"timeout is negative");
return;
}
for (j = 1; j < c->argc-1; j++) { for (j = 1; j < c->argc-1; j++) {
o = lookupKeyWrite(c->db,c->argv[j]); o = lookupKeyWrite(c->db,c->argv[j]);
if (o != NULL) { if (o != NULL) {
...@@ -811,7 +822,7 @@ void blockingPopGenericCommand(redisClient *c, int where) { ...@@ -811,7 +822,7 @@ void blockingPopGenericCommand(redisClient *c, int where) {
* "real" command will add the last element (the value) * "real" command will add the last element (the value)
* for us. If this souds like an hack to you it's just * for us. If this souds like an hack to you it's just
* because it is... */ * because it is... */
addReplySds(c,sdsnew("*2\r\n")); addReplyMultiBulkLen(c,2);
addReplyBulk(c,argv[1]); addReplyBulk(c,argv[1]);
popGenericCommand(c,where); popGenericCommand(c,where);
...@@ -823,8 +834,16 @@ void blockingPopGenericCommand(redisClient *c, int where) { ...@@ -823,8 +834,16 @@ void blockingPopGenericCommand(redisClient *c, int where) {
} }
} }
} }
/* If we are inside a MULTI/EXEC and the list is empty the only thing
* we can do is treating it as a timeout (even with timeout 0). */
if (c->flags & REDIS_MULTI) {
addReply(c,shared.nullmultibulk);
return;
}
/* If the list is empty or the key does not exists we must block */ /* If the list is empty or the key does not exists we must block */
timeout = strtol(c->argv[c->argc-1]->ptr,NULL,10); timeout = lltimeout;
if (timeout > 0) timeout += time(NULL); if (timeout > 0) timeout += time(NULL);
blockForKeys(c,c->argv+1,c->argc-2,timeout); blockForKeys(c,c->argv+1,c->argc-2,timeout);
} }
......
...@@ -4,12 +4,182 @@ ...@@ -4,12 +4,182 @@
* Set Commands * Set Commands
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
/* Factory method to return a set that *can* hold "value". When the object has
* an integer-encodable value, an intset will be returned. Otherwise a regular
* hash table. */
robj *setTypeCreate(robj *value) {
if (isObjectRepresentableAsLongLong(value,NULL) == REDIS_OK)
return createIntsetObject();
return createSetObject();
}
int setTypeAdd(robj *subject, robj *value) {
long long llval;
if (subject->encoding == REDIS_ENCODING_HT) {
if (dictAdd(subject->ptr,value,NULL) == DICT_OK) {
incrRefCount(value);
return 1;
}
} else if (subject->encoding == REDIS_ENCODING_INTSET) {
if (isObjectRepresentableAsLongLong(value,&llval) == REDIS_OK) {
uint8_t success = 0;
subject->ptr = intsetAdd(subject->ptr,llval,&success);
if (success) {
/* Convert to regular set when the intset contains
* too many entries. */
if (intsetLen(subject->ptr) > server.set_max_intset_entries)
setTypeConvert(subject,REDIS_ENCODING_HT);
return 1;
}
} else {
/* Failed to get integer from object, convert to regular set. */
setTypeConvert(subject,REDIS_ENCODING_HT);
/* The set *was* an intset and this value is not integer
* encodable, so dictAdd should always work. */
redisAssert(dictAdd(subject->ptr,value,NULL) == DICT_OK);
incrRefCount(value);
return 1;
}
} else {
redisPanic("Unknown set encoding");
}
return 0;
}
int setTypeRemove(robj *subject, robj *value) {
long long llval;
if (subject->encoding == REDIS_ENCODING_HT) {
if (dictDelete(subject->ptr,value) == DICT_OK) {
if (htNeedsResize(subject->ptr)) dictResize(subject->ptr);
return 1;
}
} else if (subject->encoding == REDIS_ENCODING_INTSET) {
if (isObjectRepresentableAsLongLong(value,&llval) == REDIS_OK) {
uint8_t success;
subject->ptr = intsetRemove(subject->ptr,llval,&success);
if (success) return 1;
}
} else {
redisPanic("Unknown set encoding");
}
return 0;
}
int setTypeIsMember(robj *subject, robj *value) {
long long llval;
if (subject->encoding == REDIS_ENCODING_HT) {
return dictFind((dict*)subject->ptr,value) != NULL;
} else if (subject->encoding == REDIS_ENCODING_INTSET) {
if (isObjectRepresentableAsLongLong(value,&llval) == REDIS_OK) {
return intsetFind((intset*)subject->ptr,llval);
}
} else {
redisPanic("Unknown set encoding");
}
return 0;
}
setTypeIterator *setTypeInitIterator(robj *subject) {
setTypeIterator *si = zmalloc(sizeof(setTypeIterator));
si->subject = subject;
si->encoding = subject->encoding;
if (si->encoding == REDIS_ENCODING_HT) {
si->di = dictGetIterator(subject->ptr);
} else if (si->encoding == REDIS_ENCODING_INTSET) {
si->ii = 0;
} else {
redisPanic("Unknown set encoding");
}
return si;
}
void setTypeReleaseIterator(setTypeIterator *si) {
if (si->encoding == REDIS_ENCODING_HT)
dictReleaseIterator(si->di);
zfree(si);
}
/* Move to the next entry in the set. Returns the object at the current
* position, or NULL when the end is reached. This object will have its
* refcount incremented, so the caller needs to take care of this. */
robj *setTypeNext(setTypeIterator *si) {
robj *ret = NULL;
if (si->encoding == REDIS_ENCODING_HT) {
dictEntry *de = dictNext(si->di);
if (de != NULL) {
ret = dictGetEntryKey(de);
incrRefCount(ret);
}
} else if (si->encoding == REDIS_ENCODING_INTSET) {
int64_t llval;
if (intsetGet(si->subject->ptr,si->ii++,&llval))
ret = createStringObjectFromLongLong(llval);
}
return ret;
}
/* Return random element from set. The returned object will always have
* an incremented refcount. */
robj *setTypeRandomElement(robj *subject) {
robj *ret = NULL;
if (subject->encoding == REDIS_ENCODING_HT) {
dictEntry *de = dictGetRandomKey(subject->ptr);
ret = dictGetEntryKey(de);
incrRefCount(ret);
} else if (subject->encoding == REDIS_ENCODING_INTSET) {
long long llval = intsetRandom(subject->ptr);
ret = createStringObjectFromLongLong(llval);
} else {
redisPanic("Unknown set encoding");
}
return ret;
}
unsigned long setTypeSize(robj *subject) {
if (subject->encoding == REDIS_ENCODING_HT) {
return dictSize((dict*)subject->ptr);
} else if (subject->encoding == REDIS_ENCODING_INTSET) {
return intsetLen((intset*)subject->ptr);
} else {
redisPanic("Unknown set encoding");
}
}
/* Convert the set to specified encoding. The resulting dict (when converting
* to a hashtable) is presized to hold the number of elements in the original
* set. */
void setTypeConvert(robj *subject, int enc) {
setTypeIterator *si;
robj *element;
redisAssert(subject->type == REDIS_SET);
if (enc == REDIS_ENCODING_HT) {
dict *d = dictCreate(&setDictType,NULL);
/* Presize the dict to avoid rehashing */
dictExpand(d,intsetLen(subject->ptr));
/* setTypeGet returns a robj with incremented refcount */
si = setTypeInitIterator(subject);
while ((element = setTypeNext(si)) != NULL)
redisAssert(dictAdd(d,element,NULL) == DICT_OK);
setTypeReleaseIterator(si);
subject->encoding = REDIS_ENCODING_HT;
zfree(subject->ptr);
subject->ptr = d;
} else {
redisPanic("Unsupported set conversion");
}
}
void saddCommand(redisClient *c) { void saddCommand(redisClient *c) {
robj *set; robj *set;
set = lookupKeyWrite(c->db,c->argv[1]); set = lookupKeyWrite(c->db,c->argv[1]);
if (set == NULL) { if (set == NULL) {
set = createSetObject(); set = setTypeCreate(c->argv[2]);
dbAdd(c->db,c->argv[1],set); dbAdd(c->db,c->argv[1],set);
} else { } else {
if (set->type != REDIS_SET) { if (set->type != REDIS_SET) {
...@@ -17,8 +187,7 @@ void saddCommand(redisClient *c) { ...@@ -17,8 +187,7 @@ void saddCommand(redisClient *c) {
return; return;
} }
} }
if (dictAdd(set->ptr,c->argv[2],NULL) == DICT_OK) { if (setTypeAdd(set,c->argv[2])) {
incrRefCount(c->argv[2]);
touchWatchedKey(c->db,c->argv[1]); touchWatchedKey(c->db,c->argv[1]);
server.dirty++; server.dirty++;
addReply(c,shared.cone); addReply(c,shared.cone);
...@@ -33,11 +202,10 @@ void sremCommand(redisClient *c) { ...@@ -33,11 +202,10 @@ void sremCommand(redisClient *c) {
if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,set,REDIS_SET)) return; checkType(c,set,REDIS_SET)) return;
if (dictDelete(set->ptr,c->argv[2]) == DICT_OK) { if (setTypeRemove(set,c->argv[2])) {
server.dirty++; if (setTypeSize(set) == 0) dbDelete(c->db,c->argv[1]);
touchWatchedKey(c->db,c->argv[1]); touchWatchedKey(c->db,c->argv[1]);
if (htNeedsResize(set->ptr)) dictResize(set->ptr); server.dirty++;
if (dictSize((dict*)set->ptr) == 0) dbDelete(c->db,c->argv[1]);
addReply(c,shared.cone); addReply(c,shared.cone);
} else { } else {
addReply(c,shared.czero); addReply(c,shared.czero);
...@@ -45,40 +213,48 @@ void sremCommand(redisClient *c) { ...@@ -45,40 +213,48 @@ void sremCommand(redisClient *c) {
} }
void smoveCommand(redisClient *c) { void smoveCommand(redisClient *c) {
robj *srcset, *dstset; robj *srcset, *dstset, *ele;
srcset = lookupKeyWrite(c->db,c->argv[1]); srcset = lookupKeyWrite(c->db,c->argv[1]);
dstset = lookupKeyWrite(c->db,c->argv[2]); dstset = lookupKeyWrite(c->db,c->argv[2]);
ele = c->argv[3];
/* If the source key does not exist return 0, if it's of the wrong type /* If the source key does not exist return 0 */
* raise an error */ if (srcset == NULL) {
if (srcset == NULL || srcset->type != REDIS_SET) { addReply(c,shared.czero);
addReply(c, srcset ? shared.wrongtypeerr : shared.czero);
return; return;
} }
/* Error if the destination key is not a set as well */
if (dstset && dstset->type != REDIS_SET) { /* If the source key has the wrong type, or the destination key
addReply(c,shared.wrongtypeerr); * is set and has the wrong type, return with an error. */
if (checkType(c,srcset,REDIS_SET) ||
(dstset && checkType(c,dstset,REDIS_SET))) return;
/* If srcset and dstset are equal, SMOVE is a no-op */
if (srcset == dstset) {
addReply(c,shared.cone);
return; return;
} }
/* Remove the element from the source set */
if (dictDelete(srcset->ptr,c->argv[3]) == DICT_ERR) { /* If the element cannot be removed from the src set, return 0. */
/* Key not found in the src set! return zero */ if (!setTypeRemove(srcset,ele)) {
addReply(c,shared.czero); addReply(c,shared.czero);
return; return;
} }
if (dictSize((dict*)srcset->ptr) == 0 && srcset != dstset)
dbDelete(c->db,c->argv[1]); /* Remove the src set from the database when empty */
if (setTypeSize(srcset) == 0) dbDelete(c->db,c->argv[1]);
touchWatchedKey(c->db,c->argv[1]); touchWatchedKey(c->db,c->argv[1]);
touchWatchedKey(c->db,c->argv[2]); touchWatchedKey(c->db,c->argv[2]);
server.dirty++; server.dirty++;
/* Add the element to the destination set */
/* Create the destination set when it doesn't exist */
if (!dstset) { if (!dstset) {
dstset = createSetObject(); dstset = setTypeCreate(ele);
dbAdd(c->db,c->argv[2],dstset); dbAdd(c->db,c->argv[2],dstset);
} }
if (dictAdd(dstset->ptr,c->argv[3],NULL) == DICT_OK)
incrRefCount(c->argv[3]); /* An extra key has changed when ele was successfully added to dstset */
if (setTypeAdd(dstset,ele)) server.dirty++;
addReply(c,shared.cone); addReply(c,shared.cone);
} }
...@@ -88,7 +264,7 @@ void sismemberCommand(redisClient *c) { ...@@ -88,7 +264,7 @@ void sismemberCommand(redisClient *c) {
if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,set,REDIS_SET)) return; checkType(c,set,REDIS_SET)) return;
if (dictFind(set->ptr,c->argv[2])) if (setTypeIsMember(set,c->argv[2]))
addReply(c,shared.cone); addReply(c,shared.cone);
else else
addReply(c,shared.czero); addReply(c,shared.czero);
...@@ -96,75 +272,64 @@ void sismemberCommand(redisClient *c) { ...@@ -96,75 +272,64 @@ void sismemberCommand(redisClient *c) {
void scardCommand(redisClient *c) { void scardCommand(redisClient *c) {
robj *o; robj *o;
dict *s;
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,o,REDIS_SET)) return; checkType(c,o,REDIS_SET)) return;
s = o->ptr; addReplyLongLong(c,setTypeSize(o));
addReplyUlong(c,dictSize(s));
} }
void spopCommand(redisClient *c) { void spopCommand(redisClient *c) {
robj *set; robj *set, *ele;
dictEntry *de;
if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.nullbulk)) == NULL || if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.nullbulk)) == NULL ||
checkType(c,set,REDIS_SET)) return; checkType(c,set,REDIS_SET)) return;
de = dictGetRandomKey(set->ptr); ele = setTypeRandomElement(set);
if (de == NULL) { if (ele == NULL) {
addReply(c,shared.nullbulk); addReply(c,shared.nullbulk);
} else { } else {
robj *ele = dictGetEntryKey(de); setTypeRemove(set,ele);
addReplyBulk(c,ele); addReplyBulk(c,ele);
dictDelete(set->ptr,ele); decrRefCount(ele);
if (htNeedsResize(set->ptr)) dictResize(set->ptr); if (setTypeSize(set) == 0) dbDelete(c->db,c->argv[1]);
if (dictSize((dict*)set->ptr) == 0) dbDelete(c->db,c->argv[1]);
touchWatchedKey(c->db,c->argv[1]); touchWatchedKey(c->db,c->argv[1]);
server.dirty++; server.dirty++;
} }
} }
void srandmemberCommand(redisClient *c) { void srandmemberCommand(redisClient *c) {
robj *set; robj *set, *ele;
dictEntry *de;
if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk)) == NULL || if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk)) == NULL ||
checkType(c,set,REDIS_SET)) return; checkType(c,set,REDIS_SET)) return;
de = dictGetRandomKey(set->ptr); ele = setTypeRandomElement(set);
if (de == NULL) { if (ele == NULL) {
addReply(c,shared.nullbulk); addReply(c,shared.nullbulk);
} else { } else {
robj *ele = dictGetEntryKey(de);
addReplyBulk(c,ele); addReplyBulk(c,ele);
decrRefCount(ele);
} }
} }
int qsortCompareSetsByCardinality(const void *s1, const void *s2) { int qsortCompareSetsByCardinality(const void *s1, const void *s2) {
dict **d1 = (void*) s1, **d2 = (void*) s2; return setTypeSize(*(robj**)s1)-setTypeSize(*(robj**)s2);
return dictSize(*d1)-dictSize(*d2);
} }
void sinterGenericCommand(redisClient *c, robj **setskeys, unsigned long setsnum, robj *dstkey) { void sinterGenericCommand(redisClient *c, robj **setkeys, unsigned long setnum, robj *dstkey) {
dict **dv = zmalloc(sizeof(dict*)*setsnum); robj **sets = zmalloc(sizeof(robj*)*setnum);
dictIterator *di; setTypeIterator *si;
dictEntry *de; robj *ele, *dstset = NULL;
robj *lenobj = NULL, *dstset = NULL; void *replylen = NULL;
unsigned long j, cardinality = 0; unsigned long j, cardinality = 0;
for (j = 0; j < setsnum; j++) { for (j = 0; j < setnum; j++) {
robj *setobj; robj *setobj = dstkey ?
lookupKeyWrite(c->db,setkeys[j]) :
setobj = dstkey ? lookupKeyRead(c->db,setkeys[j]);
lookupKeyWrite(c->db,setskeys[j]) :
lookupKeyRead(c->db,setskeys[j]);
if (!setobj) { if (!setobj) {
zfree(dv); zfree(sets);
if (dstkey) { if (dstkey) {
if (dbDelete(c->db,dstkey)) { if (dbDelete(c->db,dstkey)) {
touchWatchedKey(c->db,dstkey); touchWatchedKey(c->db,dstkey);
...@@ -176,16 +341,15 @@ void sinterGenericCommand(redisClient *c, robj **setskeys, unsigned long setsnum ...@@ -176,16 +341,15 @@ void sinterGenericCommand(redisClient *c, robj **setskeys, unsigned long setsnum
} }
return; return;
} }
if (setobj->type != REDIS_SET) { if (checkType(c,setobj,REDIS_SET)) {
zfree(dv); zfree(sets);
addReply(c,shared.wrongtypeerr);
return; return;
} }
dv[j] = setobj->ptr; sets[j] = setobj;
} }
/* Sort sets from the smallest to largest, this will improve our /* Sort sets from the smallest to largest, this will improve our
* algorithm's performace */ * algorithm's performace */
qsort(dv,setsnum,sizeof(dict*),qsortCompareSetsByCardinality); qsort(sets,setnum,sizeof(robj*),qsortCompareSetsByCardinality);
/* The first thing we should output is the total number of elements... /* The first thing we should output is the total number of elements...
* since this is a multi-bulk write, but at this stage we don't know * since this is a multi-bulk write, but at this stage we don't know
...@@ -193,45 +357,41 @@ void sinterGenericCommand(redisClient *c, robj **setskeys, unsigned long setsnum ...@@ -193,45 +357,41 @@ void sinterGenericCommand(redisClient *c, robj **setskeys, unsigned long setsnum
* to the output list and save the pointer to later modify it with the * to the output list and save the pointer to later modify it with the
* right length */ * right length */
if (!dstkey) { if (!dstkey) {
lenobj = createObject(REDIS_STRING,NULL); replylen = addDeferredMultiBulkLength(c);
addReply(c,lenobj);
decrRefCount(lenobj);
} else { } else {
/* If we have a target key where to store the resulting set /* If we have a target key where to store the resulting set
* create this key with an empty set inside */ * create this key with an empty set inside */
dstset = createSetObject(); dstset = createIntsetObject();
} }
/* Iterate all the elements of the first (smallest) set, and test /* Iterate all the elements of the first (smallest) set, and test
* the element against all the other sets, if at least one set does * the element against all the other sets, if at least one set does
* not include the element it is discarded */ * not include the element it is discarded */
di = dictGetIterator(dv[0]); si = setTypeInitIterator(sets[0]);
while((ele = setTypeNext(si)) != NULL) {
for (j = 1; j < setnum; j++)
if (!setTypeIsMember(sets[j],ele)) break;
while((de = dictNext(di)) != NULL) { /* Only take action when all sets contain the member */
robj *ele; if (j == setnum) {
for (j = 1; j < setsnum; j++)
if (dictFind(dv[j],dictGetEntryKey(de)) == NULL) break;
if (j != setsnum)
continue; /* at least one set does not contain the member */
ele = dictGetEntryKey(de);
if (!dstkey) { if (!dstkey) {
addReplyBulk(c,ele); addReplyBulk(c,ele);
cardinality++; cardinality++;
} else { } else {
dictAdd(dstset->ptr,ele,NULL); setTypeAdd(dstset,ele);
incrRefCount(ele); }
} }
decrRefCount(ele);
} }
dictReleaseIterator(di); setTypeReleaseIterator(si);
if (dstkey) { if (dstkey) {
/* Store the resulting set into the target, if the intersection /* Store the resulting set into the target, if the intersection
* is not an empty set. */ * is not an empty set. */
dbDelete(c->db,dstkey); dbDelete(c->db,dstkey);
if (dictSize((dict*)dstset->ptr) > 0) { if (setTypeSize(dstset) > 0) {
dbAdd(c->db,dstkey,dstset); dbAdd(c->db,dstkey,dstset);
addReplyLongLong(c,dictSize((dict*)dstset->ptr)); addReplyLongLong(c,setTypeSize(dstset));
} else { } else {
decrRefCount(dstset); decrRefCount(dstset);
addReply(c,shared.czero); addReply(c,shared.czero);
...@@ -239,9 +399,9 @@ void sinterGenericCommand(redisClient *c, robj **setskeys, unsigned long setsnum ...@@ -239,9 +399,9 @@ void sinterGenericCommand(redisClient *c, robj **setskeys, unsigned long setsnum
touchWatchedKey(c->db,dstkey); touchWatchedKey(c->db,dstkey);
server.dirty++; server.dirty++;
} else { } else {
lenobj->ptr = sdscatprintf(sdsempty(),"*%lu\r\n",cardinality); setDeferredMultiBulkLength(c,replylen,cardinality);
} }
zfree(dv); zfree(sets);
} }
void sinterCommand(redisClient *c) { void sinterCommand(redisClient *c) {
...@@ -252,85 +412,78 @@ void sinterstoreCommand(redisClient *c) { ...@@ -252,85 +412,78 @@ void sinterstoreCommand(redisClient *c) {
sinterGenericCommand(c,c->argv+2,c->argc-2,c->argv[1]); sinterGenericCommand(c,c->argv+2,c->argc-2,c->argv[1]);
} }
void sunionDiffGenericCommand(redisClient *c, robj **setskeys, int setsnum, robj *dstkey, int op) { #define REDIS_OP_UNION 0
dict **dv = zmalloc(sizeof(dict*)*setsnum); #define REDIS_OP_DIFF 1
dictIterator *di; #define REDIS_OP_INTER 2
dictEntry *de;
robj *dstset = NULL;
int j, cardinality = 0;
for (j = 0; j < setsnum; j++) { void sunionDiffGenericCommand(redisClient *c, robj **setkeys, int setnum, robj *dstkey, int op) {
robj *setobj; robj **sets = zmalloc(sizeof(robj*)*setnum);
setTypeIterator *si;
robj *ele, *dstset = NULL;
int j, cardinality = 0;
setobj = dstkey ? for (j = 0; j < setnum; j++) {
lookupKeyWrite(c->db,setskeys[j]) : robj *setobj = dstkey ?
lookupKeyRead(c->db,setskeys[j]); lookupKeyWrite(c->db,setkeys[j]) :
lookupKeyRead(c->db,setkeys[j]);
if (!setobj) { if (!setobj) {
dv[j] = NULL; sets[j] = NULL;
continue; continue;
} }
if (setobj->type != REDIS_SET) { if (checkType(c,setobj,REDIS_SET)) {
zfree(dv); zfree(sets);
addReply(c,shared.wrongtypeerr);
return; return;
} }
dv[j] = setobj->ptr; sets[j] = setobj;
} }
/* We need a temp set object to store our union. If the dstkey /* We need a temp set object to store our union. If the dstkey
* is not NULL (that is, we are inside an SUNIONSTORE operation) then * is not NULL (that is, we are inside an SUNIONSTORE operation) then
* this set object will be the resulting object to set into the target key*/ * this set object will be the resulting object to set into the target key*/
dstset = createSetObject(); dstset = createIntsetObject();
/* Iterate all the elements of all the sets, add every element a single /* Iterate all the elements of all the sets, add every element a single
* time to the result set */ * time to the result set */
for (j = 0; j < setsnum; j++) { for (j = 0; j < setnum; j++) {
if (op == REDIS_OP_DIFF && j == 0 && !dv[j]) break; /* result set is empty */ if (op == REDIS_OP_DIFF && j == 0 && !sets[j]) break; /* result set is empty */
if (!dv[j]) continue; /* non existing keys are like empty sets */ if (!sets[j]) continue; /* non existing keys are like empty sets */
di = dictGetIterator(dv[j]);
while((de = dictNext(di)) != NULL) { si = setTypeInitIterator(sets[j]);
robj *ele; while((ele = setTypeNext(si)) != NULL) {
/* dictAdd will not add the same element multiple times */
ele = dictGetEntryKey(de);
if (op == REDIS_OP_UNION || j == 0) { if (op == REDIS_OP_UNION || j == 0) {
if (dictAdd(dstset->ptr,ele,NULL) == DICT_OK) { if (setTypeAdd(dstset,ele)) {
incrRefCount(ele);
cardinality++; cardinality++;
} }
} else if (op == REDIS_OP_DIFF) { } else if (op == REDIS_OP_DIFF) {
if (dictDelete(dstset->ptr,ele) == DICT_OK) { if (setTypeRemove(dstset,ele)) {
cardinality--; cardinality--;
} }
} }
decrRefCount(ele);
} }
dictReleaseIterator(di); setTypeReleaseIterator(si);
/* result set is empty? Exit asap. */ /* Exit when result set is empty. */
if (op == REDIS_OP_DIFF && cardinality == 0) break; if (op == REDIS_OP_DIFF && cardinality == 0) break;
} }
/* Output the content of the resulting set, if not in STORE mode */ /* Output the content of the resulting set, if not in STORE mode */
if (!dstkey) { if (!dstkey) {
addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",cardinality)); addReplyMultiBulkLen(c,cardinality);
di = dictGetIterator(dstset->ptr); si = setTypeInitIterator(dstset);
while((de = dictNext(di)) != NULL) { while((ele = setTypeNext(si)) != NULL) {
robj *ele;
ele = dictGetEntryKey(de);
addReplyBulk(c,ele); addReplyBulk(c,ele);
decrRefCount(ele);
} }
dictReleaseIterator(di); setTypeReleaseIterator(si);
decrRefCount(dstset); decrRefCount(dstset);
} else { } else {
/* If we have a target key where to store the resulting set /* If we have a target key where to store the resulting set
* create this key with the result set inside */ * create this key with the result set inside */
dbDelete(c->db,dstkey); dbDelete(c->db,dstkey);
if (dictSize((dict*)dstset->ptr) > 0) { if (setTypeSize(dstset) > 0) {
dbAdd(c->db,dstkey,dstset); dbAdd(c->db,dstkey,dstset);
addReplyLongLong(c,dictSize((dict*)dstset->ptr)); addReplyLongLong(c,setTypeSize(dstset));
} else { } else {
decrRefCount(dstset); decrRefCount(dstset);
addReply(c,shared.czero); addReply(c,shared.czero);
...@@ -338,7 +491,7 @@ void sunionDiffGenericCommand(redisClient *c, robj **setskeys, int setsnum, robj ...@@ -338,7 +491,7 @@ void sunionDiffGenericCommand(redisClient *c, robj **setskeys, int setsnum, robj
touchWatchedKey(c->db,dstkey); touchWatchedKey(c->db,dstkey);
server.dirty++; server.dirty++;
} }
zfree(dv); zfree(sets);
} }
void sunionCommand(redisClient *c) { void sunionCommand(redisClient *c) {
......
...@@ -12,12 +12,11 @@ void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expir ...@@ -12,12 +12,11 @@ void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expir
if (getLongFromObjectOrReply(c, expire, &seconds, NULL) != REDIS_OK) if (getLongFromObjectOrReply(c, expire, &seconds, NULL) != REDIS_OK)
return; return;
if (seconds <= 0) { if (seconds <= 0) {
addReplySds(c,sdsnew("-ERR invalid expire time in SETEX\r\n")); addReplyError(c,"invalid expire time in SETEX");
return; return;
} }
} }
if (nx) deleteIfVolatile(c->db,key);
retval = dbAdd(c->db,key,val); retval = dbAdd(c->db,key,val);
if (retval == REDIS_ERR) { if (retval == REDIS_ERR) {
if (!nx) { if (!nx) {
...@@ -80,7 +79,7 @@ void getsetCommand(redisClient *c) { ...@@ -80,7 +79,7 @@ void getsetCommand(redisClient *c) {
void mgetCommand(redisClient *c) { void mgetCommand(redisClient *c) {
int j; int j;
addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",c->argc-1)); addReplyMultiBulkLen(c,c->argc-1);
for (j = 1; j < c->argc; j++) { for (j = 1; j < c->argc; j++) {
robj *o = lookupKeyRead(c->db,c->argv[j]); robj *o = lookupKeyRead(c->db,c->argv[j]);
if (o == NULL) { if (o == NULL) {
...@@ -99,7 +98,7 @@ void msetGenericCommand(redisClient *c, int nx) { ...@@ -99,7 +98,7 @@ void msetGenericCommand(redisClient *c, int nx) {
int j, busykeys = 0; int j, busykeys = 0;
if ((c->argc % 2) == 0) { if ((c->argc % 2) == 0) {
addReplySds(c,sdsnew("-ERR wrong number of arguments for MSET\r\n")); addReplyError(c,"wrong number of arguments for MSET");
return; return;
} }
/* Handle the NX flag. The MSETNX semantic is to return zero and don't /* Handle the NX flag. The MSETNX semantic is to return zero and don't
...@@ -212,7 +211,7 @@ void appendCommand(redisClient *c) { ...@@ -212,7 +211,7 @@ void appendCommand(redisClient *c) {
} }
touchWatchedKey(c->db,c->argv[1]); touchWatchedKey(c->db,c->argv[1]);
server.dirty++; server.dirty++;
addReplySds(c,sdscatprintf(sdsempty(),":%lu\r\n",(unsigned long)totlen)); addReplyLongLong(c,totlen);
} }
void substrCommand(redisClient *c) { void substrCommand(redisClient *c) {
......
...@@ -24,13 +24,7 @@ ...@@ -24,13 +24,7 @@
* from tail to head, useful for ZREVRANGE. */ * from tail to head, useful for ZREVRANGE. */
zskiplistNode *zslCreateNode(int level, double score, robj *obj) { zskiplistNode *zslCreateNode(int level, double score, robj *obj) {
zskiplistNode *zn = zmalloc(sizeof(*zn)); zskiplistNode *zn = zmalloc(sizeof(*zn)+level*sizeof(struct zskiplistLevel));
zn->forward = zmalloc(sizeof(zskiplistNode*) * level);
if (level > 1)
zn->span = zmalloc(sizeof(unsigned int) * (level - 1));
else
zn->span = NULL;
zn->score = score; zn->score = score;
zn->obj = obj; zn->obj = obj;
return zn; return zn;
...@@ -45,11 +39,8 @@ zskiplist *zslCreate(void) { ...@@ -45,11 +39,8 @@ zskiplist *zslCreate(void) {
zsl->length = 0; zsl->length = 0;
zsl->header = zslCreateNode(ZSKIPLIST_MAXLEVEL,0,NULL); zsl->header = zslCreateNode(ZSKIPLIST_MAXLEVEL,0,NULL);
for (j = 0; j < ZSKIPLIST_MAXLEVEL; j++) { for (j = 0; j < ZSKIPLIST_MAXLEVEL; j++) {
zsl->header->forward[j] = NULL; zsl->header->level[j].forward = NULL;
zsl->header->level[j].span = 0;
/* span has space for ZSKIPLIST_MAXLEVEL-1 elements */
if (j < ZSKIPLIST_MAXLEVEL-1)
zsl->header->span[j] = 0;
} }
zsl->header->backward = NULL; zsl->header->backward = NULL;
zsl->tail = NULL; zsl->tail = NULL;
...@@ -58,19 +49,15 @@ zskiplist *zslCreate(void) { ...@@ -58,19 +49,15 @@ zskiplist *zslCreate(void) {
void zslFreeNode(zskiplistNode *node) { void zslFreeNode(zskiplistNode *node) {
decrRefCount(node->obj); decrRefCount(node->obj);
zfree(node->forward);
zfree(node->span);
zfree(node); zfree(node);
} }
void zslFree(zskiplist *zsl) { void zslFree(zskiplist *zsl) {
zskiplistNode *node = zsl->header->forward[0], *next; zskiplistNode *node = zsl->header->level[0].forward, *next;
zfree(zsl->header->forward);
zfree(zsl->header->span);
zfree(zsl->header); zfree(zsl->header);
while(node) { while(node) {
next = node->forward[0]; next = node->level[0].forward;
zslFreeNode(node); zslFreeNode(node);
node = next; node = next;
} }
...@@ -84,7 +71,7 @@ int zslRandomLevel(void) { ...@@ -84,7 +71,7 @@ int zslRandomLevel(void) {
return (level<ZSKIPLIST_MAXLEVEL) ? level : ZSKIPLIST_MAXLEVEL; return (level<ZSKIPLIST_MAXLEVEL) ? level : ZSKIPLIST_MAXLEVEL;
} }
void zslInsert(zskiplist *zsl, double score, robj *obj) { zskiplistNode *zslInsert(zskiplist *zsl, double score, robj *obj) {
zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x; zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x;
unsigned int rank[ZSKIPLIST_MAXLEVEL]; unsigned int rank[ZSKIPLIST_MAXLEVEL];
int i, level; int i, level;
...@@ -93,13 +80,12 @@ void zslInsert(zskiplist *zsl, double score, robj *obj) { ...@@ -93,13 +80,12 @@ void zslInsert(zskiplist *zsl, double score, robj *obj) {
for (i = zsl->level-1; i >= 0; i--) { for (i = zsl->level-1; i >= 0; i--) {
/* store rank that is crossed to reach the insert position */ /* store rank that is crossed to reach the insert position */
rank[i] = i == (zsl->level-1) ? 0 : rank[i+1]; rank[i] = i == (zsl->level-1) ? 0 : rank[i+1];
while (x->level[i].forward &&
while (x->forward[i] && (x->level[i].forward->score < score ||
(x->forward[i]->score < score || (x->level[i].forward->score == score &&
(x->forward[i]->score == score && compareStringObjects(x->level[i].forward->obj,obj) < 0))) {
compareStringObjects(x->forward[i]->obj,obj) < 0))) { rank[i] += x->level[i].span;
rank[i] += i > 0 ? x->span[i-1] : 1; x = x->level[i].forward;
x = x->forward[i];
} }
update[i] = x; update[i] = x;
} }
...@@ -112,56 +98,51 @@ void zslInsert(zskiplist *zsl, double score, robj *obj) { ...@@ -112,56 +98,51 @@ void zslInsert(zskiplist *zsl, double score, robj *obj) {
for (i = zsl->level; i < level; i++) { for (i = zsl->level; i < level; i++) {
rank[i] = 0; rank[i] = 0;
update[i] = zsl->header; update[i] = zsl->header;
update[i]->span[i-1] = zsl->length; update[i]->level[i].span = zsl->length;
} }
zsl->level = level; zsl->level = level;
} }
x = zslCreateNode(level,score,obj); x = zslCreateNode(level,score,obj);
for (i = 0; i < level; i++) { for (i = 0; i < level; i++) {
x->forward[i] = update[i]->forward[i]; x->level[i].forward = update[i]->level[i].forward;
update[i]->forward[i] = x; update[i]->level[i].forward = x;
/* update span covered by update[i] as x is inserted here */ /* update span covered by update[i] as x is inserted here */
if (i > 0) { x->level[i].span = update[i]->level[i].span - (rank[0] - rank[i]);
x->span[i-1] = update[i]->span[i-1] - (rank[0] - rank[i]); update[i]->level[i].span = (rank[0] - rank[i]) + 1;
update[i]->span[i-1] = (rank[0] - rank[i]) + 1;
}
} }
/* increment span for untouched levels */ /* increment span for untouched levels */
for (i = level; i < zsl->level; i++) { for (i = level; i < zsl->level; i++) {
update[i]->span[i-1]++; update[i]->level[i].span++;
} }
x->backward = (update[0] == zsl->header) ? NULL : update[0]; x->backward = (update[0] == zsl->header) ? NULL : update[0];
if (x->forward[0]) if (x->level[0].forward)
x->forward[0]->backward = x; x->level[0].forward->backward = x;
else else
zsl->tail = x; zsl->tail = x;
zsl->length++; zsl->length++;
return x;
} }
/* Internal function used by zslDelete, zslDeleteByScore and zslDeleteByRank */ /* Internal function used by zslDelete, zslDeleteByScore and zslDeleteByRank */
void zslDeleteNode(zskiplist *zsl, zskiplistNode *x, zskiplistNode **update) { void zslDeleteNode(zskiplist *zsl, zskiplistNode *x, zskiplistNode **update) {
int i; int i;
for (i = 0; i < zsl->level; i++) { for (i = 0; i < zsl->level; i++) {
if (update[i]->forward[i] == x) { if (update[i]->level[i].forward == x) {
if (i > 0) { update[i]->level[i].span += x->level[i].span - 1;
update[i]->span[i-1] += x->span[i-1] - 1; update[i]->level[i].forward = x->level[i].forward;
}
update[i]->forward[i] = x->forward[i];
} else { } else {
/* invariant: i > 0, because update[0]->forward[0] update[i]->level[i].span -= 1;
* is always equal to x */
update[i]->span[i-1] -= 1;
} }
} }
if (x->forward[0]) { if (x->level[0].forward) {
x->forward[0]->backward = x->backward; x->level[0].forward->backward = x->backward;
} else { } else {
zsl->tail = x->backward; zsl->tail = x->backward;
} }
while(zsl->level > 1 && zsl->header->forward[zsl->level-1] == NULL) while(zsl->level > 1 && zsl->header->level[zsl->level-1].forward == NULL)
zsl->level--; zsl->level--;
zsl->length--; zsl->length--;
} }
...@@ -173,16 +154,16 @@ int zslDelete(zskiplist *zsl, double score, robj *obj) { ...@@ -173,16 +154,16 @@ int zslDelete(zskiplist *zsl, double score, robj *obj) {
x = zsl->header; x = zsl->header;
for (i = zsl->level-1; i >= 0; i--) { for (i = zsl->level-1; i >= 0; i--) {
while (x->forward[i] && while (x->level[i].forward &&
(x->forward[i]->score < score || (x->level[i].forward->score < score ||
(x->forward[i]->score == score && (x->level[i].forward->score == score &&
compareStringObjects(x->forward[i]->obj,obj) < 0))) compareStringObjects(x->level[i].forward->obj,obj) < 0)))
x = x->forward[i]; x = x->level[i].forward;
update[i] = x; update[i] = x;
} }
/* We may have multiple elements with the same score, what we need /* We may have multiple elements with the same score, what we need
* is to find the element with both the right score and object. */ * is to find the element with both the right score and object. */
x = x->forward[0]; x = x->level[0].forward;
if (x && score == x->score && equalStringObjects(x->obj,obj)) { if (x && score == x->score && equalStringObjects(x->obj,obj)) {
zslDeleteNode(zsl, x, update); zslDeleteNode(zsl, x, update);
zslFreeNode(x); zslFreeNode(x);
...@@ -204,16 +185,16 @@ unsigned long zslDeleteRangeByScore(zskiplist *zsl, double min, double max, dict ...@@ -204,16 +185,16 @@ unsigned long zslDeleteRangeByScore(zskiplist *zsl, double min, double max, dict
x = zsl->header; x = zsl->header;
for (i = zsl->level-1; i >= 0; i--) { for (i = zsl->level-1; i >= 0; i--) {
while (x->forward[i] && x->forward[i]->score < min) while (x->level[i].forward && x->level[i].forward->score < min)
x = x->forward[i]; x = x->level[i].forward;
update[i] = x; update[i] = x;
} }
/* We may have multiple elements with the same score, what we need /* We may have multiple elements with the same score, what we need
* is to find the element with both the right score and object. */ * is to find the element with both the right score and object. */
x = x->forward[0]; x = x->level[0].forward;
while (x && x->score <= max) { while (x && x->score <= max) {
zskiplistNode *next = x->forward[0]; zskiplistNode *next = x->level[0].forward;
zslDeleteNode(zsl, x, update); zslDeleteNode(zsl,x,update);
dictDelete(dict,x->obj); dictDelete(dict,x->obj);
zslFreeNode(x); zslFreeNode(x);
removed++; removed++;
...@@ -231,18 +212,18 @@ unsigned long zslDeleteRangeByRank(zskiplist *zsl, unsigned int start, unsigned ...@@ -231,18 +212,18 @@ unsigned long zslDeleteRangeByRank(zskiplist *zsl, unsigned int start, unsigned
x = zsl->header; x = zsl->header;
for (i = zsl->level-1; i >= 0; i--) { for (i = zsl->level-1; i >= 0; i--) {
while (x->forward[i] && (traversed + (i > 0 ? x->span[i-1] : 1)) < start) { while (x->level[i].forward && (traversed + x->level[i].span) < start) {
traversed += i > 0 ? x->span[i-1] : 1; traversed += x->level[i].span;
x = x->forward[i]; x = x->level[i].forward;
} }
update[i] = x; update[i] = x;
} }
traversed++; traversed++;
x = x->forward[0]; x = x->level[0].forward;
while (x && traversed <= end) { while (x && traversed <= end) {
zskiplistNode *next = x->forward[0]; zskiplistNode *next = x->level[0].forward;
zslDeleteNode(zsl, x, update); zslDeleteNode(zsl,x,update);
dictDelete(dict,x->obj); dictDelete(dict,x->obj);
zslFreeNode(x); zslFreeNode(x);
removed++; removed++;
...@@ -260,12 +241,12 @@ zskiplistNode *zslFirstWithScore(zskiplist *zsl, double score) { ...@@ -260,12 +241,12 @@ zskiplistNode *zslFirstWithScore(zskiplist *zsl, double score) {
x = zsl->header; x = zsl->header;
for (i = zsl->level-1; i >= 0; i--) { for (i = zsl->level-1; i >= 0; i--) {
while (x->forward[i] && x->forward[i]->score < score) while (x->level[i].forward && x->level[i].forward->score < score)
x = x->forward[i]; x = x->level[i].forward;
} }
/* We may have multiple elements with the same score, what we need /* We may have multiple elements with the same score, what we need
* is to find the element with both the right score and object. */ * is to find the element with both the right score and object. */
return x->forward[0]; return x->level[0].forward;
} }
/* Find the rank for an element by both score and key. /* Find the rank for an element by both score and key.
...@@ -279,12 +260,12 @@ unsigned long zslistTypeGetRank(zskiplist *zsl, double score, robj *o) { ...@@ -279,12 +260,12 @@ unsigned long zslistTypeGetRank(zskiplist *zsl, double score, robj *o) {
x = zsl->header; x = zsl->header;
for (i = zsl->level-1; i >= 0; i--) { for (i = zsl->level-1; i >= 0; i--) {
while (x->forward[i] && while (x->level[i].forward &&
(x->forward[i]->score < score || (x->level[i].forward->score < score ||
(x->forward[i]->score == score && (x->level[i].forward->score == score &&
compareStringObjects(x->forward[i]->obj,o) <= 0))) { compareStringObjects(x->level[i].forward->obj,o) <= 0))) {
rank += i > 0 ? x->span[i-1] : 1; rank += x->level[i].span;
x = x->forward[i]; x = x->level[i].forward;
} }
/* x might be equal to zsl->header, so test if obj is non-NULL */ /* x might be equal to zsl->header, so test if obj is non-NULL */
...@@ -303,10 +284,10 @@ zskiplistNode* zslistTypeGetElementByRank(zskiplist *zsl, unsigned long rank) { ...@@ -303,10 +284,10 @@ zskiplistNode* zslistTypeGetElementByRank(zskiplist *zsl, unsigned long rank) {
x = zsl->header; x = zsl->header;
for (i = zsl->level-1; i >= 0; i--) { for (i = zsl->level-1; i >= 0; i--) {
while (x->forward[i] && (traversed + (i>0 ? x->span[i-1] : 1)) <= rank) while (x->level[i].forward && (traversed + x->level[i].span) <= rank)
{ {
traversed += i > 0 ? x->span[i-1] : 1; traversed += x->level[i].span;
x = x->forward[i]; x = x->level[i].forward;
} }
if (traversed == rank) { if (traversed == rank) {
return x; return x;
...@@ -319,13 +300,11 @@ zskiplistNode* zslistTypeGetElementByRank(zskiplist *zsl, unsigned long rank) { ...@@ -319,13 +300,11 @@ zskiplistNode* zslistTypeGetElementByRank(zskiplist *zsl, unsigned long rank) {
* Sorted set commands * Sorted set commands
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
/* This generic command implements both ZADD and ZINCRBY. /* This generic command implements both ZADD and ZINCRBY. */
* scoreval is the score if the operation is a ZADD (doincrement == 0) or void zaddGenericCommand(redisClient *c, robj *key, robj *ele, double score, int incr) {
* the increment if the operation is a ZINCRBY (doincrement == 1). */
void zaddGenericCommand(redisClient *c, robj *key, robj *ele, double scoreval, int doincrement) {
robj *zsetobj; robj *zsetobj;
zset *zs; zset *zs;
double *score; zskiplistNode *znode;
zsetobj = lookupKeyWrite(c->db,key); zsetobj = lookupKeyWrite(c->db,key);
if (zsetobj == NULL) { if (zsetobj == NULL) {
...@@ -339,72 +318,72 @@ void zaddGenericCommand(redisClient *c, robj *key, robj *ele, double scoreval, i ...@@ -339,72 +318,72 @@ void zaddGenericCommand(redisClient *c, robj *key, robj *ele, double scoreval, i
} }
zs = zsetobj->ptr; zs = zsetobj->ptr;
/* Ok now since we implement both ZADD and ZINCRBY here the code /* Since both ZADD and ZINCRBY are implemented here, we need to increment
* needs to handle the two different conditions. It's all about setting * the score first by the current score if ZINCRBY is called. */
* '*score', that is, the new score to set, to the right value. */ if (incr) {
score = zmalloc(sizeof(double));
if (doincrement) {
dictEntry *de;
/* Read the old score. If the element was not present starts from 0 */ /* Read the old score. If the element was not present starts from 0 */
de = dictFind(zs->dict,ele); dictEntry *de = dictFind(zs->dict,ele);
if (de) { if (de != NULL)
double *oldscore = dictGetEntryVal(de); score += *(double*)dictGetEntryVal(de);
*score = *oldscore + scoreval;
} else { if (isnan(score)) {
*score = scoreval; addReplyError(c,"resulting score is not a number (NaN)");
}
if (isnan(*score)) {
addReplySds(c,
sdsnew("-ERR resulting score is not a number (NaN)\r\n"));
zfree(score);
/* Note that we don't need to check if the zset may be empty and /* Note that we don't need to check if the zset may be empty and
* should be removed here, as we can only obtain Nan as score if * should be removed here, as we can only obtain Nan as score if
* there was already an element in the sorted set. */ * there was already an element in the sorted set. */
return; return;
} }
} else {
*score = scoreval;
} }
/* What follows is a simple remove and re-insert operation that is common /* We need to remove and re-insert the element when it was already present
* to both ZADD and ZINCRBY... */ * in the dictionary, to update the skiplist. Note that we delay adding a
if (dictAdd(zs->dict,ele,score) == DICT_OK) { * pointer to the score because we want to reference the score in the
/* case 1: New element */ * skiplist node. */
if (dictAdd(zs->dict,ele,NULL) == DICT_OK) {
dictEntry *de;
/* New element */
incrRefCount(ele); /* added to hash */ incrRefCount(ele); /* added to hash */
zslInsert(zs->zsl,*score,ele); znode = zslInsert(zs->zsl,score,ele);
incrRefCount(ele); /* added to skiplist */ incrRefCount(ele); /* added to skiplist */
/* Update the score in the dict entry */
de = dictFind(zs->dict,ele);
redisAssert(de != NULL);
dictGetEntryVal(de) = &znode->score;
touchWatchedKey(c->db,c->argv[1]); touchWatchedKey(c->db,c->argv[1]);
server.dirty++; server.dirty++;
if (doincrement) if (incr)
addReplyDouble(c,*score); addReplyDouble(c,score);
else else
addReply(c,shared.cone); addReply(c,shared.cone);
} else { } else {
dictEntry *de; dictEntry *de;
double *oldscore; robj *curobj;
double *curscore;
int deleted;
/* case 2: Score update operation */ /* Update score */
de = dictFind(zs->dict,ele); de = dictFind(zs->dict,ele);
redisAssert(de != NULL); redisAssert(de != NULL);
oldscore = dictGetEntryVal(de); curobj = dictGetEntryKey(de);
if (*score != *oldscore) { curscore = dictGetEntryVal(de);
int deleted;
/* Remove and insert the element in the skip list with new score */ /* When the score is updated, reuse the existing string object to
deleted = zslDelete(zs->zsl,*oldscore,ele); * prevent extra alloc/dealloc of strings on ZINCRBY. */
if (score != *curscore) {
deleted = zslDelete(zs->zsl,*curscore,curobj);
redisAssert(deleted != 0); redisAssert(deleted != 0);
zslInsert(zs->zsl,*score,ele); znode = zslInsert(zs->zsl,score,curobj);
incrRefCount(ele); incrRefCount(curobj);
/* Update the score in the hash table */
dictReplace(zs->dict,ele,score); /* Update the score in the current dict entry */
dictGetEntryVal(de) = &znode->score;
touchWatchedKey(c->db,c->argv[1]); touchWatchedKey(c->db,c->argv[1]);
server.dirty++; server.dirty++;
} else {
zfree(score);
} }
if (doincrement) if (incr)
addReplyDouble(c,*score); addReplyDouble(c,score);
else else
addReply(c,shared.czero); addReply(c,shared.czero);
} }
...@@ -426,7 +405,7 @@ void zremCommand(redisClient *c) { ...@@ -426,7 +405,7 @@ void zremCommand(redisClient *c) {
robj *zsetobj; robj *zsetobj;
zset *zs; zset *zs;
dictEntry *de; dictEntry *de;
double *oldscore; double curscore;
int deleted; int deleted;
if ((zsetobj = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || if ((zsetobj = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
...@@ -439,8 +418,8 @@ void zremCommand(redisClient *c) { ...@@ -439,8 +418,8 @@ void zremCommand(redisClient *c) {
return; return;
} }
/* Delete from the skiplist */ /* Delete from the skiplist */
oldscore = dictGetEntryVal(de); curscore = *(double*)dictGetEntryVal(de);
deleted = zslDelete(zs->zsl,*oldscore,c->argv[2]); deleted = zslDelete(zs->zsl,curscore,c->argv[2]);
redisAssert(deleted != 0); redisAssert(deleted != 0);
/* Delete from the hash table */ /* Delete from the hash table */
...@@ -554,6 +533,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { ...@@ -554,6 +533,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
zsetopsrc *src; zsetopsrc *src;
robj *dstobj; robj *dstobj;
zset *dstzset; zset *dstzset;
zskiplistNode *znode;
dictIterator *di; dictIterator *di;
dictEntry *de; dictEntry *de;
int touched = 0; int touched = 0;
...@@ -561,7 +541,8 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { ...@@ -561,7 +541,8 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
/* expect setnum input keys to be given */ /* expect setnum input keys to be given */
setnum = atoi(c->argv[2]->ptr); setnum = atoi(c->argv[2]->ptr);
if (setnum < 1) { if (setnum < 1) {
addReplySds(c,sdsnew("-ERR at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE\r\n")); addReplyError(c,
"at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE");
return; return;
} }
...@@ -644,28 +625,26 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { ...@@ -644,28 +625,26 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
* from small to large, all src[i > 0].dict are non-empty too */ * from small to large, all src[i > 0].dict are non-empty too */
di = dictGetIterator(src[0].dict); di = dictGetIterator(src[0].dict);
while((de = dictNext(di)) != NULL) { while((de = dictNext(di)) != NULL) {
double *score = zmalloc(sizeof(double)), value; double score, value;
*score = src[0].weight * zunionInterDictValue(de);
score = src[0].weight * zunionInterDictValue(de);
for (j = 1; j < setnum; j++) { for (j = 1; j < setnum; j++) {
dictEntry *other = dictFind(src[j].dict,dictGetEntryKey(de)); dictEntry *other = dictFind(src[j].dict,dictGetEntryKey(de));
if (other) { if (other) {
value = src[j].weight * zunionInterDictValue(other); value = src[j].weight * zunionInterDictValue(other);
zunionInterAggregate(score, value, aggregate); zunionInterAggregate(&score, value, aggregate);
} else { } else {
break; break;
} }
} }
/* skip entry when not present in every source dict */ /* accept entry only when present in every source dict */
if (j != setnum) { if (j == setnum) {
zfree(score);
} else {
robj *o = dictGetEntryKey(de); robj *o = dictGetEntryKey(de);
dictAdd(dstzset->dict,o,score); znode = zslInsert(dstzset->zsl,score,o);
incrRefCount(o); /* added to dictionary */
zslInsert(dstzset->zsl,*score,o);
incrRefCount(o); /* added to skiplist */ incrRefCount(o); /* added to skiplist */
dictAdd(dstzset->dict,o,&znode->score);
incrRefCount(o); /* added to dictionary */
} }
} }
dictReleaseIterator(di); dictReleaseIterator(di);
...@@ -676,11 +655,12 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { ...@@ -676,11 +655,12 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
di = dictGetIterator(src[i].dict); di = dictGetIterator(src[i].dict);
while((de = dictNext(di)) != NULL) { while((de = dictNext(di)) != NULL) {
/* skip key when already processed */ double score, value;
if (dictFind(dstzset->dict,dictGetEntryKey(de)) != NULL) continue;
double *score = zmalloc(sizeof(double)), value; /* skip key when already processed */
*score = src[i].weight * zunionInterDictValue(de); if (dictFind(dstzset->dict,dictGetEntryKey(de)) != NULL)
continue;
score = src[i].weight * zunionInterDictValue(de);
/* because the zsets are sorted by size, its only possible /* because the zsets are sorted by size, its only possible
* for sets at larger indices to hold this entry */ * for sets at larger indices to hold this entry */
...@@ -688,15 +668,15 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { ...@@ -688,15 +668,15 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
dictEntry *other = dictFind(src[j].dict,dictGetEntryKey(de)); dictEntry *other = dictFind(src[j].dict,dictGetEntryKey(de));
if (other) { if (other) {
value = src[j].weight * zunionInterDictValue(other); value = src[j].weight * zunionInterDictValue(other);
zunionInterAggregate(score, value, aggregate); zunionInterAggregate(&score, value, aggregate);
} }
} }
robj *o = dictGetEntryKey(de); robj *o = dictGetEntryKey(de);
dictAdd(dstzset->dict,o,score); znode = zslInsert(dstzset->zsl,score,o);
incrRefCount(o); /* added to dictionary */
zslInsert(dstzset->zsl,*score,o);
incrRefCount(o); /* added to skiplist */ incrRefCount(o); /* added to skiplist */
dictAdd(dstzset->dict,o,&znode->score);
incrRefCount(o); /* added to dictionary */
} }
dictReleaseIterator(di); dictReleaseIterator(di);
} }
...@@ -778,18 +758,17 @@ void zrangeGenericCommand(redisClient *c, int reverse) { ...@@ -778,18 +758,17 @@ void zrangeGenericCommand(redisClient *c, int reverse) {
ln = start == 0 ? zsl->tail : zslistTypeGetElementByRank(zsl, llen-start); ln = start == 0 ? zsl->tail : zslistTypeGetElementByRank(zsl, llen-start);
} else { } else {
ln = start == 0 ? ln = start == 0 ?
zsl->header->forward[0] : zslistTypeGetElementByRank(zsl, start+1); zsl->header->level[0].forward : zslistTypeGetElementByRank(zsl, start+1);
} }
/* Return the result in form of a multi-bulk reply */ /* Return the result in form of a multi-bulk reply */
addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n", addReplyMultiBulkLen(c,withscores ? (rangelen*2) : rangelen);
withscores ? (rangelen*2) : rangelen));
for (j = 0; j < rangelen; j++) { for (j = 0; j < rangelen; j++) {
ele = ln->obj; ele = ln->obj;
addReplyBulk(c,ele); addReplyBulk(c,ele);
if (withscores) if (withscores)
addReplyDouble(c,ln->score); addReplyDouble(c,ln->score);
ln = reverse ? ln->backward : ln->forward[0]; ln = reverse ? ln->backward : ln->level[0].forward;
} }
} }
...@@ -840,8 +819,7 @@ void genericZrangebyscoreCommand(redisClient *c, int justcount) { ...@@ -840,8 +819,7 @@ void genericZrangebyscoreCommand(redisClient *c, int justcount) {
if (c->argc != (4 + withscores) && c->argc != (7 + withscores)) if (c->argc != (4 + withscores) && c->argc != (7 + withscores))
badsyntax = 1; badsyntax = 1;
if (badsyntax) { if (badsyntax) {
addReplySds(c, addReplyError(c,"wrong number of arguments for ZRANGEBYSCORE");
sdsnew("-ERR wrong number of arguments for ZRANGEBYSCORE\r\n"));
return; return;
} }
...@@ -866,13 +844,14 @@ void genericZrangebyscoreCommand(redisClient *c, int justcount) { ...@@ -866,13 +844,14 @@ void genericZrangebyscoreCommand(redisClient *c, int justcount) {
zset *zsetobj = o->ptr; zset *zsetobj = o->ptr;
zskiplist *zsl = zsetobj->zsl; zskiplist *zsl = zsetobj->zsl;
zskiplistNode *ln; zskiplistNode *ln;
robj *ele, *lenobj = NULL; robj *ele;
void *replylen = NULL;
unsigned long rangelen = 0; unsigned long rangelen = 0;
/* Get the first node with the score >= min, or with /* Get the first node with the score >= min, or with
* score > min if 'minex' is true. */ * score > min if 'minex' is true. */
ln = zslFirstWithScore(zsl,min); ln = zslFirstWithScore(zsl,min);
while (minex && ln && ln->score == min) ln = ln->forward[0]; while (minex && ln && ln->score == min) ln = ln->level[0].forward;
if (ln == NULL) { if (ln == NULL) {
/* No element matching the speciifed interval */ /* No element matching the speciifed interval */
...@@ -884,16 +863,13 @@ void genericZrangebyscoreCommand(redisClient *c, int justcount) { ...@@ -884,16 +863,13 @@ void genericZrangebyscoreCommand(redisClient *c, int justcount) {
* are in the list, so we push this object that will represent * are in the list, so we push this object that will represent
* the multi-bulk length in the output buffer, and will "fix" * the multi-bulk length in the output buffer, and will "fix"
* it later */ * it later */
if (!justcount) { if (!justcount)
lenobj = createObject(REDIS_STRING,NULL); replylen = addDeferredMultiBulkLength(c);
addReply(c,lenobj);
decrRefCount(lenobj);
}
while(ln && (maxex ? (ln->score < max) : (ln->score <= max))) { while(ln && (maxex ? (ln->score < max) : (ln->score <= max))) {
if (offset) { if (offset) {
offset--; offset--;
ln = ln->forward[0]; ln = ln->level[0].forward;
continue; continue;
} }
if (limit == 0) break; if (limit == 0) break;
...@@ -903,14 +879,14 @@ void genericZrangebyscoreCommand(redisClient *c, int justcount) { ...@@ -903,14 +879,14 @@ void genericZrangebyscoreCommand(redisClient *c, int justcount) {
if (withscores) if (withscores)
addReplyDouble(c,ln->score); addReplyDouble(c,ln->score);
} }
ln = ln->forward[0]; ln = ln->level[0].forward;
rangelen++; rangelen++;
if (limit > 0) limit--; if (limit > 0) limit--;
} }
if (justcount) { if (justcount) {
addReplyLongLong(c,(long)rangelen); addReplyLongLong(c,(long)rangelen);
} else { } else {
lenobj->ptr = sdscatprintf(sdsempty(),"*%lu\r\n", setDeferredMultiBulkLength(c,replylen,
withscores ? (rangelen*2) : rangelen); withscores ? (rangelen*2) : rangelen);
} }
} }
...@@ -933,7 +909,7 @@ void zcardCommand(redisClient *c) { ...@@ -933,7 +909,7 @@ void zcardCommand(redisClient *c) {
checkType(c,o,REDIS_ZSET)) return; checkType(c,o,REDIS_ZSET)) return;
zs = o->ptr; zs = o->ptr;
addReplyUlong(c,zs->zsl->length); addReplyLongLong(c,zs->zsl->length);
} }
void zscoreCommand(redisClient *c) { void zscoreCommand(redisClient *c) {
......
/* This is a really minimal testing framework for C.
*
* Example:
*
* test_cond("Check if 1 == 1", 1==1)
* test_cond("Check if 5 > 10", 5 > 10)
* test_report()
*
* Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __TESTHELP_H
#define __TESTHELP_H
int __failed_tests = 0;
int __test_num = 0;
#define test_cond(descr,_c) do { \
__test_num++; printf("%d - %s: ", __test_num, descr); \
if(_c) printf("PASSED\n"); else {printf("FAILED\n"); __failed_tests++;} \
} while(0);
#define test_report() do { \
printf("%d tests, %d passed, %d failed\n", __test_num, \
__test_num-__failed_tests, __failed_tests); \
if (__failed_tests) { \
printf("=== WARNING === We have failed tests here...\n"); \
} \
} while(0);
#endif
...@@ -200,24 +200,44 @@ int ll2string(char *s, size_t len, long long value) { ...@@ -200,24 +200,44 @@ int ll2string(char *s, size_t len, long long value) {
return l; return l;
} }
/* Check if the nul-terminated string 's' can be represented by a long /* Check if the sds string 's' can be represented by a long long
* (that is, is a number that fits into long without any other space or * (that is, is a number that fits into long without any other space or
* character before or after the digits). * character before or after the digits, so that converting this number
* back to a string will result in the same bytes as the original string).
* *
* If so, the function returns REDIS_OK and *longval is set to the value * If so, the function returns REDIS_OK and *llongval is set to the value
* of the number. Otherwise REDIS_ERR is returned */ * of the number. Otherwise REDIS_ERR is returned */
int isStringRepresentableAsLong(sds s, long *longval) { int isStringRepresentableAsLongLong(sds s, long long *llongval) {
char buf[32], *endptr; char buf[32], *endptr;
long value; long long value;
int slen; int slen;
value = strtol(s, &endptr, 10); value = strtoll(s, &endptr, 10);
if (endptr[0] != '\0') return REDIS_ERR; if (endptr[0] != '\0') return REDIS_ERR;
slen = ll2string(buf,32,value); slen = ll2string(buf,32,value);
/* If the number converted back into a string is not identical /* If the number converted back into a string is not identical
* then it's not possible to encode the string as integer */ * then it's not possible to encode the string as integer */
if (sdslen(s) != (unsigned)slen || memcmp(buf,s,slen)) return REDIS_ERR; if (sdslen(s) != (unsigned)slen || memcmp(buf,s,slen)) return REDIS_ERR;
if (longval) *longval = value; if (llongval) *llongval = value;
return REDIS_OK;
}
int isStringRepresentableAsLong(sds s, long *longval) {
long long ll;
if (isStringRepresentableAsLongLong(s,&ll) == REDIS_ERR) return REDIS_ERR;
if (ll < LONG_MIN || ll > LONG_MAX) return REDIS_ERR;
*longval = (long)ll;
return REDIS_OK; return REDIS_OK;
} }
int isObjectRepresentableAsLongLong(robj *o, long long *llongval) {
redisAssert(o->type == REDIS_STRING);
if (o->encoding == REDIS_ENCODING_INT) {
if (llongval) *llongval = (long) o->ptr;
return REDIS_OK;
} else {
return isStringRepresentableAsLongLong(o->ptr,llongval);
}
}
#define REDIS_VERSION "2.1.2" #define REDIS_VERSION "2.1.4"
...@@ -110,6 +110,11 @@ void vmInit(void) { ...@@ -110,6 +110,11 @@ void vmInit(void) {
/* LZF requires a lot of stack */ /* LZF requires a lot of stack */
pthread_attr_init(&server.io_threads_attr); pthread_attr_init(&server.io_threads_attr);
pthread_attr_getstacksize(&server.io_threads_attr, &stacksize); pthread_attr_getstacksize(&server.io_threads_attr, &stacksize);
/* Solaris may report a stacksize of 0, let's set it to 1 otherwise
* multiplying it by 2 in the while loop later will not really help ;) */
if (!stacksize) stacksize = 1;
while (stacksize < REDIS_THREAD_STACK_SIZE) stacksize *= 2; while (stacksize < REDIS_THREAD_STACK_SIZE) stacksize *= 2;
pthread_attr_setstacksize(&server.io_threads_attr, stacksize); pthread_attr_setstacksize(&server.io_threads_attr, stacksize);
/* Listen for events in the threaded I/O pipe */ /* Listen for events in the threaded I/O pipe */
...@@ -395,6 +400,10 @@ double computeObjectSwappability(robj *o) { ...@@ -395,6 +400,10 @@ double computeObjectSwappability(robj *o) {
z = (o->type == REDIS_ZSET); z = (o->type == REDIS_ZSET);
d = z ? ((zset*)o->ptr)->dict : o->ptr; d = z ? ((zset*)o->ptr)->dict : o->ptr;
if (!z && o->encoding == REDIS_ENCODING_INTSET) {
intset *is = o->ptr;
asize = sizeof(*is)+is->encoding*is->length;
} else {
asize = sizeof(dict)+(sizeof(struct dictEntry*)*dictSlots(d)); asize = sizeof(dict)+(sizeof(struct dictEntry*)*dictSlots(d));
if (z) asize += sizeof(zset)-sizeof(dict); if (z) asize += sizeof(zset)-sizeof(dict);
if (dictSize(d)) { if (dictSize(d)) {
...@@ -405,6 +414,7 @@ double computeObjectSwappability(robj *o) { ...@@ -405,6 +414,7 @@ double computeObjectSwappability(robj *o) {
asize += (sizeof(struct dictEntry)+elesize)*dictSize(d); asize += (sizeof(struct dictEntry)+elesize)*dictSize(d);
if (z) asize += sizeof(zskiplistNode)*dictSize(d); if (z) asize += sizeof(zskiplistNode)*dictSize(d);
} }
}
break; break;
case REDIS_HASH: case REDIS_HASH:
if (o->encoding == REDIS_ENCODING_ZIPMAP) { if (o->encoding == REDIS_ENCODING_ZIPMAP) {
...@@ -543,7 +553,15 @@ void freeIOJob(iojob *j) { ...@@ -543,7 +553,15 @@ void freeIOJob(iojob *j) {
/* Every time a thread finished a Job, it writes a byte into the write side /* Every time a thread finished a Job, it writes a byte into the write side
* of an unix pipe in order to "awake" the main thread, and this function * of an unix pipe in order to "awake" the main thread, and this function
* is called. */ * is called.
*
* Note that this is called both by the event loop, when a I/O thread
* sends a byte in the notification pipe, and is also directly called from
* waitEmptyIOJobsQueue().
*
* In the latter case we don't want to swap more, so we use the
* "privdata" argument setting it to a not NULL value to signal this
* condition. */
void vmThreadedIOCompletedJob(aeEventLoop *el, int fd, void *privdata, void vmThreadedIOCompletedJob(aeEventLoop *el, int fd, void *privdata,
int mask) int mask)
{ {
...@@ -553,6 +571,8 @@ void vmThreadedIOCompletedJob(aeEventLoop *el, int fd, void *privdata, ...@@ -553,6 +571,8 @@ void vmThreadedIOCompletedJob(aeEventLoop *el, int fd, void *privdata,
REDIS_NOTUSED(mask); REDIS_NOTUSED(mask);
REDIS_NOTUSED(privdata); REDIS_NOTUSED(privdata);
if (privdata != NULL) trytoswap = 0; /* check the comments above... */
/* For every byte we read in the read side of the pipe, there is one /* For every byte we read in the read side of the pipe, there is one
* I/O job completed to process. */ * I/O job completed to process. */
while((retval = read(fd,buf,1)) == 1) { while((retval = read(fd,buf,1)) == 1) {
...@@ -864,7 +884,8 @@ void waitEmptyIOJobsQueue(void) { ...@@ -864,7 +884,8 @@ void waitEmptyIOJobsQueue(void) {
io_processed_len = listLength(server.io_processed); io_processed_len = listLength(server.io_processed);
unlockThreadedIO(); unlockThreadedIO();
if (io_processed_len) { if (io_processed_len) {
vmThreadedIOCompletedJob(NULL,server.io_ready_pipe_read,NULL,0); vmThreadedIOCompletedJob(NULL,server.io_ready_pipe_read,
(void*)0xdeadbeef,0);
usleep(1000); /* 1 millisecond */ usleep(1000); /* 1 millisecond */
} else { } else {
usleep(10000); /* 10 milliseconds */ usleep(10000); /* 10 milliseconds */
......
/* Memory layout of a ziplist, containing "foo", "bar", "quux": /* The ziplist is a specially encoded dually linked list that is designed
* <zlbytes><zllen><len>"foo"<len>"bar"<len>"quux" * to be very memory efficient. It stores both strings and integer values,
* where integers are encoded as actual integers instead of a series of
* characters. It allows push and pop operations on either side of the list
* in O(1) time. However, because every operation requires a reallocation of
* the memory used by the ziplist, the actual complexity is related to the
* amount of memory used by the ziplist.
* *
* <zlbytes> is an unsigned integer to hold the number of bytes that * ----------------------------------------------------------------------------
* the ziplist occupies. This is stored to not have to traverse the ziplist
* to know the new length when pushing.
* *
* <zllen> is the number of items in the ziplist. When this value is * ZIPLIST OVERALL LAYOUT:
* greater than 254, we need to traverse the entire list to know * The general layout of the ziplist is as follows:
* how many items it holds. * <zlbytes><zltail><zllen><entry><entry><zlend>
* *
* <len> is the number of bytes occupied by a single entry. When this * <zlbytes> is an unsigned integer to hold the number of bytes that the
* number is greater than 253, the length will occupy 5 bytes, where * ziplist occupies. This value needs to be stored to be able to resize the
* the extra bytes contain an unsigned integer to hold the length. * entire structure without the need to traverse it first.
*
* <zltail> is the offset to the last entry in the list. This allows a pop
* operation on the far side of the list without the need for full traversal.
*
* <zllen> is the number of entries.When this value is larger than 2**16-2,
* we need to traverse the entire list to know how many items it holds.
*
* <zlend> is a single byte special value, equal to 255, which indicates the
* end of the list.
*
* ZIPLIST ENTRIES:
* Every entry in the ziplist is prefixed by a header that contains two pieces
* of information. First, the length of the previous entry is stored to be
* able to traverse the list from back to front. Second, the encoding with an
* optional string length of the entry itself is stored.
*
* The length of the previous entry is encoded in the following way:
* If this length is smaller than 254 bytes, it will only consume a single
* byte that takes the length as value. When the length is greater than or
* equal to 254, it will consume 5 bytes. The first byte is set to 254 to
* indicate a larger value is following. The remaining 4 bytes take the
* length of the previous entry as value.
*
* The other header field of the entry itself depends on the contents of the
* entry. When the entry is a string, the first 2 bits of this header will hold
* the type of encoding used to store the length of the string, followed by the
* actual length of the string. When the entry is an integer the first 2 bits
* are both set to 1. The following 2 bits are used to specify what kind of
* integer will be stored after this header. An overview of the different
* types and encodings is as follows:
*
* |00pppppp| - 1 byte
* String value with length less than or equal to 63 bytes (6 bits).
* |01pppppp|qqqqqqqq| - 2 bytes
* String value with length less than or equal to 16383 bytes (14 bits).
* |10______|qqqqqqqq|rrrrrrrr|ssssssss|tttttttt| - 5 bytes
* String value with length greater than or equal to 16384 bytes.
* |1100____| - 1 byte
* Integer encoded as int16_t (2 bytes).
* |1101____| - 1 byte
* Integer encoded as int32_t (4 bytes).
* |1110____| - 1 byte
* Integer encoded as int64_t (8 bytes).
*/ */
#include <stdio.h> #include <stdio.h>
...@@ -25,25 +71,20 @@ ...@@ -25,25 +71,20 @@
int ll2string(char *s, size_t len, long long value); int ll2string(char *s, size_t len, long long value);
/* Important note: the ZIP_END value is used to depict the end of the
* ziplist structure. When a pointer contains an entry, the first couple
* of bytes contain the encoded length of the previous entry. This length
* is encoded as ZIP_ENC_RAW length, so the first two bits will contain 00
* and the byte will therefore never have a value of 255. */
#define ZIP_END 255 #define ZIP_END 255
#define ZIP_BIGLEN 254 #define ZIP_BIGLEN 254
/* Entry encoding */ /* Different encoding/length possibilities */
#define ZIP_ENC_RAW 0 #define ZIP_STR_06B (0 << 6)
#define ZIP_ENC_INT16 1 #define ZIP_STR_14B (1 << 6)
#define ZIP_ENC_INT32 2 #define ZIP_STR_32B (2 << 6)
#define ZIP_ENC_INT64 3 #define ZIP_INT_16B (0xc0 | 0<<4)
#define ZIP_ENCODING(p) ((p)[0] >> 6) #define ZIP_INT_32B (0xc0 | 1<<4)
#define ZIP_INT_64B (0xc0 | 2<<4)
/* Length encoding for raw entries */ /* Macro's to determine type */
#define ZIP_LEN_INLINE 0 #define ZIP_IS_STR(enc) (((enc) & 0xc0) < 0xc0)
#define ZIP_LEN_UINT16 1 #define ZIP_IS_INT(enc) (!ZIP_IS_STR(enc) && ((enc) & 0x30) < 0x30)
#define ZIP_LEN_UINT32 2
/* Utility macros */ /* Utility macros */
#define ZIPLIST_BYTES(zl) (*((uint32_t*)(zl))) #define ZIPLIST_BYTES(zl) (*((uint32_t*)(zl)))
...@@ -67,14 +108,25 @@ typedef struct zlentry { ...@@ -67,14 +108,25 @@ typedef struct zlentry {
unsigned char *p; unsigned char *p;
} zlentry; } zlentry;
/* Return the encoding pointer to by 'p'. */
static unsigned int zipEntryEncoding(unsigned char *p) {
/* String encoding: 2 MSBs */
unsigned char b = p[0] & 0xc0;
if (b < 0xc0) {
return b;
} else {
/* Integer encoding: 4 MSBs */
return p[0] & 0xf0;
}
assert(NULL);
}
/* Return bytes needed to store integer encoded by 'encoding' */ /* Return bytes needed to store integer encoded by 'encoding' */
static unsigned int zipEncodingSize(unsigned char encoding) { static unsigned int zipIntSize(unsigned char encoding) {
if (encoding == ZIP_ENC_INT16) { switch(encoding) {
return sizeof(int16_t); case ZIP_INT_16B: return sizeof(int16_t);
} else if (encoding == ZIP_ENC_INT32) { case ZIP_INT_32B: return sizeof(int32_t);
return sizeof(int32_t); case ZIP_INT_64B: return sizeof(int64_t);
} else if (encoding == ZIP_ENC_INT64) {
return sizeof(int64_t);
} }
assert(NULL); assert(NULL);
} }
...@@ -82,23 +134,28 @@ static unsigned int zipEncodingSize(unsigned char encoding) { ...@@ -82,23 +134,28 @@ static unsigned int zipEncodingSize(unsigned char encoding) {
/* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is /* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is
* provided, it is set to the number of bytes required to encode the length. */ * provided, it is set to the number of bytes required to encode the length. */
static unsigned int zipDecodeLength(unsigned char *p, unsigned int *lensize) { static unsigned int zipDecodeLength(unsigned char *p, unsigned int *lensize) {
unsigned char encoding = ZIP_ENCODING(p), lenenc; unsigned char encoding = zipEntryEncoding(p);
unsigned int len; unsigned int len;
if (encoding == ZIP_ENC_RAW) { if (ZIP_IS_STR(encoding)) {
lenenc = (p[0] >> 4) & 0x3; switch(encoding) {
if (lenenc == ZIP_LEN_INLINE) { case ZIP_STR_06B:
len = p[0] & 0xf; len = p[0] & 0x3f;
if (lensize) *lensize = 1; if (lensize) *lensize = 1;
} else if (lenenc == ZIP_LEN_UINT16) { break;
len = p[1] | (p[2] << 8); case ZIP_STR_14B:
if (lensize) *lensize = 3; len = ((p[0] & 0x3f) << 8) | p[1];
} else { if (lensize) *lensize = 2;
len = p[1] | (p[2] << 8) | (p[3] << 16) | (p[4] << 24); break;
case ZIP_STR_32B:
len = (p[1] << 24) | (p[2] << 16) | (p[3] << 8) | p[4];
if (lensize) *lensize = 5; if (lensize) *lensize = 5;
break;
default:
assert(NULL);
} }
} else { } else {
len = zipEncodingSize(encoding); len = zipIntSize(encoding);
if (lensize) *lensize = 1; if (lensize) *lensize = 1;
} }
return len; return len;
...@@ -106,34 +163,36 @@ static unsigned int zipDecodeLength(unsigned char *p, unsigned int *lensize) { ...@@ -106,34 +163,36 @@ static unsigned int zipDecodeLength(unsigned char *p, unsigned int *lensize) {
/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns /* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
* the amount of bytes required to encode such a length. */ * the amount of bytes required to encode such a length. */
static unsigned int zipEncodeLength(unsigned char *p, char encoding, unsigned int rawlen) { static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, unsigned int rawlen) {
unsigned char len = 1, lenenc, buf[5]; unsigned char len = 1, buf[5];
if (encoding == ZIP_ENC_RAW) {
if (rawlen <= 0xf) { if (ZIP_IS_STR(encoding)) {
/* Although encoding is given it may not be set for strings,
* so we determine it here using the raw length. */
if (rawlen <= 0x3f) {
if (!p) return len; if (!p) return len;
lenenc = ZIP_LEN_INLINE; buf[0] = ZIP_STR_06B | rawlen;
buf[0] = rawlen; } else if (rawlen <= 0x3fff) {
} else if (rawlen <= 0xffff) { len += 1;
len += 2;
if (!p) return len; if (!p) return len;
lenenc = ZIP_LEN_UINT16; buf[0] = ZIP_STR_14B | ((rawlen >> 8) & 0x3f);
buf[1] = (rawlen ) & 0xff; buf[1] = rawlen & 0xff;
buf[2] = (rawlen >> 8) & 0xff;
} else { } else {
len += 4; len += 4;
if (!p) return len; if (!p) return len;
lenenc = ZIP_LEN_UINT32; buf[0] = ZIP_STR_32B;
buf[1] = (rawlen ) & 0xff; buf[1] = (rawlen >> 24) & 0xff;
buf[2] = (rawlen >> 8) & 0xff; buf[2] = (rawlen >> 16) & 0xff;
buf[3] = (rawlen >> 16) & 0xff; buf[3] = (rawlen >> 8) & 0xff;
buf[4] = (rawlen >> 24) & 0xff; buf[4] = rawlen & 0xff;
}
buf[0] = (lenenc << 4) | (buf[0] & 0xf);
} }
} else {
/* Implies integer encoding, so length is always 1. */
if (!p) return len; if (!p) return len;
buf[0] = encoding;
}
/* Apparently we need to store the length in 'p' */ /* Store this length at p */
buf[0] = (encoding << 6) | (buf[0] & 0x3f);
memcpy(p,buf,len); memcpy(p,buf,len);
return len; return len;
} }
...@@ -167,6 +226,14 @@ static unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) { ...@@ -167,6 +226,14 @@ static unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) {
} }
} }
/* Encode the length of the previous entry and write it to "p". This only
* uses the larger encoding (required in __ziplistCascadeUpdate). */
static void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) {
if (p == NULL) return;
p[0] = ZIP_BIGLEN;
memcpy(p+1,&len,sizeof(len));
}
/* Return the difference in number of bytes needed to store the new length /* Return the difference in number of bytes needed to store the new length
* "len" on the entry pointed to by "p". */ * "len" on the entry pointed to by "p". */
static int zipPrevLenByteDiff(unsigned char *p, unsigned int len) { static int zipPrevLenByteDiff(unsigned char *p, unsigned int len) {
...@@ -198,11 +265,11 @@ static int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long ...@@ -198,11 +265,11 @@ static int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long
/* Great, the string can be encoded. Check what's the smallest /* Great, the string can be encoded. Check what's the smallest
* of our encoding types that can hold this value. */ * of our encoding types that can hold this value. */
if (value >= INT16_MIN && value <= INT16_MAX) { if (value >= INT16_MIN && value <= INT16_MAX) {
*encoding = ZIP_ENC_INT16; *encoding = ZIP_INT_16B;
} else if (value >= INT32_MIN && value <= INT32_MAX) { } else if (value >= INT32_MIN && value <= INT32_MAX) {
*encoding = ZIP_ENC_INT32; *encoding = ZIP_INT_32B;
} else { } else {
*encoding = ZIP_ENC_INT64; *encoding = ZIP_INT_64B;
} }
*v = value; *v = value;
return 1; return 1;
...@@ -215,13 +282,13 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi ...@@ -215,13 +282,13 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi
int16_t i16; int16_t i16;
int32_t i32; int32_t i32;
int64_t i64; int64_t i64;
if (encoding == ZIP_ENC_INT16) { if (encoding == ZIP_INT_16B) {
i16 = value; i16 = value;
memcpy(p,&i16,sizeof(i16)); memcpy(p,&i16,sizeof(i16));
} else if (encoding == ZIP_ENC_INT32) { } else if (encoding == ZIP_INT_32B) {
i32 = value; i32 = value;
memcpy(p,&i32,sizeof(i32)); memcpy(p,&i32,sizeof(i32));
} else if (encoding == ZIP_ENC_INT64) { } else if (encoding == ZIP_INT_64B) {
i64 = value; i64 = value;
memcpy(p,&i64,sizeof(i64)); memcpy(p,&i64,sizeof(i64));
} else { } else {
...@@ -234,13 +301,13 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { ...@@ -234,13 +301,13 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) {
int16_t i16; int16_t i16;
int32_t i32; int32_t i32;
int64_t i64, ret; int64_t i64, ret;
if (encoding == ZIP_ENC_INT16) { if (encoding == ZIP_INT_16B) {
memcpy(&i16,p,sizeof(i16)); memcpy(&i16,p,sizeof(i16));
ret = i16; ret = i16;
} else if (encoding == ZIP_ENC_INT32) { } else if (encoding == ZIP_INT_32B) {
memcpy(&i32,p,sizeof(i32)); memcpy(&i32,p,sizeof(i32));
ret = i32; ret = i32;
} else if (encoding == ZIP_ENC_INT64) { } else if (encoding == ZIP_INT_64B) {
memcpy(&i64,p,sizeof(i64)); memcpy(&i64,p,sizeof(i64));
ret = i64; ret = i64;
} else { } else {
...@@ -255,7 +322,7 @@ static zlentry zipEntry(unsigned char *p) { ...@@ -255,7 +322,7 @@ static zlentry zipEntry(unsigned char *p) {
e.prevrawlen = zipPrevDecodeLength(p,&e.prevrawlensize); e.prevrawlen = zipPrevDecodeLength(p,&e.prevrawlensize);
e.len = zipDecodeLength(p+e.prevrawlensize,&e.lensize); e.len = zipDecodeLength(p+e.prevrawlensize,&e.lensize);
e.headersize = e.prevrawlensize+e.lensize; e.headersize = e.prevrawlensize+e.lensize;
e.encoding = ZIP_ENCODING(p+e.prevrawlensize); e.encoding = zipEntryEncoding(p+e.prevrawlensize);
e.p = p; e.p = p;
return e; return e;
} }
...@@ -285,11 +352,86 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) { ...@@ -285,11 +352,86 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
return zl; return zl;
} }
/* When an entry is inserted, we need to set the prevlen field of the next
* entry to equal the length of the inserted entry. It can occur that this
* length cannot be encoded in 1 byte and the next entry needs to be grow
* a bit larger to hold the 5-byte encoded prevlen. This can be done for free,
* because this only happens when an entry is already being inserted (which
* causes a realloc and memmove). However, encoding the prevlen may require
* that this entry is grown as well. This effect may cascade throughout
* the ziplist when there are consecutive entries with a size close to
* ZIP_BIGLEN, so we need to check that the prevlen can be encoded in every
* consecutive entry.
*
* Note that this effect can also happen in reverse, where the bytes required
* to encode the prevlen field can shrink. This effect is deliberately ignored,
* because it can cause a "flapping" effect where a chain prevlen fields is
* first grown and then shrunk again after consecutive inserts. Rather, the
* field is allowed to stay larger than necessary, because a large prevlen
* field implies the ziplist is holding large entries anyway.
*
* The pointer "p" points to the first entry that does NOT need to be
* updated, i.e. consecutive fields MAY need an update. */
static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) {
unsigned int curlen = ZIPLIST_BYTES(zl), rawlen, rawlensize;
unsigned int offset, noffset, extra;
unsigned char *np;
zlentry cur, next;
while (p[0] != ZIP_END) {
cur = zipEntry(p);
rawlen = cur.headersize + cur.len;
rawlensize = zipPrevEncodeLength(NULL,rawlen);
/* Abort if there is no next entry. */
if (p[rawlen] == ZIP_END) break;
next = zipEntry(p+rawlen);
/* Abort when "prevlen" has not changed. */
if (next.prevrawlen == rawlen) break;
if (next.prevrawlensize < rawlensize) {
/* The "prevlen" field of "next" needs more bytes to hold
* the raw length of "cur". */
offset = p-zl;
extra = rawlensize-next.prevrawlensize;
zl = ziplistResize(zl,curlen+extra);
ZIPLIST_TAIL_OFFSET(zl) += extra;
p = zl+offset;
/* Move the tail to the back. */
np = p+rawlen;
noffset = np-zl;
memmove(np+rawlensize,
np+next.prevrawlensize,
curlen-noffset-next.prevrawlensize-1);
zipPrevEncodeLength(np,rawlen);
/* Advance the cursor */
p += rawlen;
} else {
if (next.prevrawlensize > rawlensize) {
/* This would result in shrinking, which we want to avoid.
* So, set "rawlen" in the available bytes. */
zipPrevEncodeLengthForceLarge(p+rawlen,rawlen);
} else {
zipPrevEncodeLength(p+rawlen,rawlen);
}
/* Stop here, as the raw length of "next" has not changed. */
break;
}
}
return zl;
}
/* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */ /* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */
static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int num) { static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int num) {
unsigned int i, totlen, deleted = 0; unsigned int i, totlen, deleted = 0;
int nextdiff = 0; int offset, nextdiff = 0;
zlentry first = zipEntry(p); zlentry first, tail;
first = zipEntry(p);
for (i = 0; p[0] != ZIP_END && i < num; i++) { for (i = 0; p[0] != ZIP_END && i < num; i++) {
p += zipRawEntryLength(p); p += zipRawEntryLength(p);
deleted++; deleted++;
...@@ -306,7 +448,14 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig ...@@ -306,7 +448,14 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
zipPrevEncodeLength(p-nextdiff,first.prevrawlen); zipPrevEncodeLength(p-nextdiff,first.prevrawlen);
/* Update offset for tail */ /* Update offset for tail */
ZIPLIST_TAIL_OFFSET(zl) -= totlen+nextdiff; ZIPLIST_TAIL_OFFSET(zl) -= totlen;
/* When the tail contains more than one entry, we need to take
* "nextdiff" in account as well. Otherwise, a change in the
* size of prevlen doesn't have an effect on the *tail* offset. */
tail = zipEntry(p);
if (p[tail.headersize+tail.len] != ZIP_END)
ZIPLIST_TAIL_OFFSET(zl) += nextdiff;
/* Move tail to the front of the ziplist */ /* Move tail to the front of the ziplist */
memmove(first.p,p-nextdiff,ZIPLIST_BYTES(zl)-(p-zl)-1+nextdiff); memmove(first.p,p-nextdiff,ZIPLIST_BYTES(zl)-(p-zl)-1+nextdiff);
...@@ -316,8 +465,15 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig ...@@ -316,8 +465,15 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
} }
/* Resize and update length */ /* Resize and update length */
offset = first.p-zl;
zl = ziplistResize(zl, ZIPLIST_BYTES(zl)-totlen+nextdiff); zl = ziplistResize(zl, ZIPLIST_BYTES(zl)-totlen+nextdiff);
ZIPLIST_INCR_LENGTH(zl,-deleted); ZIPLIST_INCR_LENGTH(zl,-deleted);
p = zl+offset;
/* When nextdiff != 0, the raw length of the next entry has changed, so
* we need to cascade the update throughout the ziplist */
if (nextdiff != 0)
zl = __ziplistCascadeUpdate(zl,p);
} }
return zl; return zl;
} }
...@@ -326,29 +482,30 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig ...@@ -326,29 +482,30 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) { static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
unsigned int curlen = ZIPLIST_BYTES(zl), reqlen, prevlen = 0; unsigned int curlen = ZIPLIST_BYTES(zl), reqlen, prevlen = 0;
unsigned int offset, nextdiff = 0; unsigned int offset, nextdiff = 0;
unsigned char *tail; unsigned char encoding = 0;
unsigned char encoding = ZIP_ENC_RAW;
long long value; long long value;
zlentry entry; zlentry entry, tail;
/* Find out prevlen for the entry that is inserted. */ /* Find out prevlen for the entry that is inserted. */
if (p[0] != ZIP_END) { if (p[0] != ZIP_END) {
entry = zipEntry(p); entry = zipEntry(p);
prevlen = entry.prevrawlen; prevlen = entry.prevrawlen;
} else { } else {
tail = ZIPLIST_ENTRY_TAIL(zl); unsigned char *ptail = ZIPLIST_ENTRY_TAIL(zl);
if (tail[0] != ZIP_END) { if (ptail[0] != ZIP_END) {
prevlen = zipRawEntryLength(tail); prevlen = zipRawEntryLength(ptail);
} }
} }
/* See if the entry can be encoded */ /* See if the entry can be encoded */
if (zipTryEncoding(s,slen,&value,&encoding)) { if (zipTryEncoding(s,slen,&value,&encoding)) {
reqlen = zipEncodingSize(encoding); /* 'encoding' is set to the appropriate integer encoding */
reqlen = zipIntSize(encoding);
} else { } else {
/* 'encoding' is untouched, however zipEncodeLength will use the
* string length to figure out how to encode it. */
reqlen = slen; reqlen = slen;
} }
/* We need space for both the length of the previous entry and /* We need space for both the length of the previous entry and
* the length of the payload. */ * the length of the payload. */
reqlen += zipPrevEncodeLength(NULL,prevlen); reqlen += zipPrevEncodeLength(NULL,prevlen);
...@@ -368,22 +525,39 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig ...@@ -368,22 +525,39 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
if (p[0] != ZIP_END) { if (p[0] != ZIP_END) {
/* Subtract one because of the ZIP_END bytes */ /* Subtract one because of the ZIP_END bytes */
memmove(p+reqlen,p-nextdiff,curlen-offset-1+nextdiff); memmove(p+reqlen,p-nextdiff,curlen-offset-1+nextdiff);
/* Encode this entry's raw length in the next entry. */ /* Encode this entry's raw length in the next entry. */
zipPrevEncodeLength(p+reqlen,reqlen); zipPrevEncodeLength(p+reqlen,reqlen);
/* Update offset for tail */ /* Update offset for tail */
ZIPLIST_TAIL_OFFSET(zl) += reqlen+nextdiff; ZIPLIST_TAIL_OFFSET(zl) += reqlen;
/* When the tail contains more than one entry, we need to take
* "nextdiff" in account as well. Otherwise, a change in the
* size of prevlen doesn't have an effect on the *tail* offset. */
tail = zipEntry(p+reqlen);
if (p[reqlen+tail.headersize+tail.len] != ZIP_END)
ZIPLIST_TAIL_OFFSET(zl) += nextdiff;
} else { } else {
/* This element will be the new tail. */ /* This element will be the new tail. */
ZIPLIST_TAIL_OFFSET(zl) = p-zl; ZIPLIST_TAIL_OFFSET(zl) = p-zl;
} }
/* When nextdiff != 0, the raw length of the next entry has changed, so
* we need to cascade the update throughout the ziplist */
if (nextdiff != 0) {
offset = p-zl;
zl = __ziplistCascadeUpdate(zl,p+reqlen);
p = zl+offset;
}
/* Write the entry */ /* Write the entry */
p += zipPrevEncodeLength(p,prevlen); p += zipPrevEncodeLength(p,prevlen);
p += zipEncodeLength(p,encoding,slen); p += zipEncodeLength(p,encoding,slen);
if (encoding != ZIP_ENC_RAW) { if (ZIP_IS_STR(encoding)) {
zipSaveInteger(p,value,encoding);
} else {
memcpy(p,s,slen); memcpy(p,s,slen);
} else {
zipSaveInteger(p,value,encoding);
} }
ZIPLIST_INCR_LENGTH(zl,1); ZIPLIST_INCR_LENGTH(zl,1);
return zl; return zl;
...@@ -449,6 +623,7 @@ unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p) { ...@@ -449,6 +623,7 @@ unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p) {
return NULL; return NULL;
} else { } else {
entry = zipEntry(p); entry = zipEntry(p);
assert(entry.prevrawlen > 0);
return p-entry.prevrawlen; return p-entry.prevrawlen;
} }
} }
...@@ -463,7 +638,7 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *sl ...@@ -463,7 +638,7 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *sl
if (sstr) *sstr = NULL; if (sstr) *sstr = NULL;
entry = zipEntry(p); entry = zipEntry(p);
if (entry.encoding == ZIP_ENC_RAW) { if (ZIP_IS_STR(entry.encoding)) {
if (sstr) { if (sstr) {
*slen = entry.len; *slen = entry.len;
*sstr = p+entry.headersize; *sstr = p+entry.headersize;
...@@ -510,7 +685,7 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int ...@@ -510,7 +685,7 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int
if (p[0] == ZIP_END) return 0; if (p[0] == ZIP_END) return 0;
entry = zipEntry(p); entry = zipEntry(p);
if (entry.encoding == ZIP_ENC_RAW) { if (ZIP_IS_STR(entry.encoding)) {
/* Raw compare */ /* Raw compare */
if (entry.len == slen) { if (entry.len == slen) {
return memcmp(p+entry.headersize,sstr,slen) == 0; return memcmp(p+entry.headersize,sstr,slen) == 0;
...@@ -554,21 +729,52 @@ unsigned int ziplistSize(unsigned char *zl) { ...@@ -554,21 +729,52 @@ unsigned int ziplistSize(unsigned char *zl) {
void ziplistRepr(unsigned char *zl) { void ziplistRepr(unsigned char *zl) {
unsigned char *p; unsigned char *p;
int index = 0;
zlentry entry; zlentry entry;
printf("{total bytes %d} {length %u}\n",ZIPLIST_BYTES(zl), ZIPLIST_LENGTH(zl)); printf(
"{total bytes %d} "
"{length %u}\n"
"{tail offset %u}\n",
ZIPLIST_BYTES(zl),
ZIPLIST_LENGTH(zl),
ZIPLIST_TAIL_OFFSET(zl));
p = ZIPLIST_ENTRY_HEAD(zl); p = ZIPLIST_ENTRY_HEAD(zl);
while(*p != ZIP_END) { while(*p != ZIP_END) {
entry = zipEntry(p); entry = zipEntry(p);
printf("{offset %ld, header %u, payload %u} ",p-zl,entry.headersize,entry.len); printf(
"{"
"addr 0x%08lx, "
"index %2d, "
"offset %5ld, "
"rl: %5u, "
"hs %2u, "
"pl: %5u, "
"pls: %2u, "
"payload %5u"
"} ",
(long unsigned int)p,
index,
p-zl,
entry.headersize+entry.len,
entry.headersize,
entry.prevrawlen,
entry.prevrawlensize,
entry.len);
p += entry.headersize; p += entry.headersize;
if (entry.encoding == ZIP_ENC_RAW) { if (ZIP_IS_STR(entry.encoding)) {
if (entry.len > 40) {
fwrite(p,40,1,stdout);
printf("...");
} else {
fwrite(p,entry.len,1,stdout); fwrite(p,entry.len,1,stdout);
}
} else { } else {
printf("%lld", (long long) zipLoadInteger(p,entry.encoding)); printf("%lld", (long long) zipLoadInteger(p,entry.encoding));
} }
printf("\n"); printf("\n");
p += entry.len; p += entry.len;
index++;
} }
printf("{end}\n\n"); printf("{end}\n\n");
} }
...@@ -664,6 +870,10 @@ int main(int argc, char **argv) { ...@@ -664,6 +870,10 @@ int main(int argc, char **argv) {
unsigned int elen; unsigned int elen;
long long value; long long value;
/* If an argument is given, use it as the random seed. */
if (argc == 2)
srand(atoi(argv[1]));
zl = createIntList(); zl = createIntList();
ziplistRepr(zl); ziplistRepr(zl);
...@@ -915,6 +1125,25 @@ int main(int argc, char **argv) { ...@@ -915,6 +1125,25 @@ int main(int argc, char **argv) {
ziplistRepr(zl); ziplistRepr(zl);
} }
printf("Regression test for >255 byte strings:\n");
{
char v1[257],v2[257];
memset(v1,'x',256);
memset(v2,'y',256);
zl = ziplistNew();
zl = ziplistPush(zl,(unsigned char*)v1,strlen(v1),ZIPLIST_TAIL);
zl = ziplistPush(zl,(unsigned char*)v2,strlen(v2),ZIPLIST_TAIL);
/* Pop values again and compare their value. */
p = ziplistIndex(zl,0);
assert(ziplistGet(p,&entry,&elen,&value));
assert(strncmp(v1,entry,elen) == 0);
p = ziplistIndex(zl,1);
assert(ziplistGet(p,&entry,&elen,&value));
assert(strncmp(v2,entry,elen) == 0);
printf("SUCCESS\n\n");
}
printf("Create long list and check indices:\n"); printf("Create long list and check indices:\n");
{ {
zl = ziplistNew(); zl = ziplistNew();
...@@ -958,7 +1187,57 @@ int main(int argc, char **argv) { ...@@ -958,7 +1187,57 @@ int main(int argc, char **argv) {
printf("ERROR: \"1025\"\n"); printf("ERROR: \"1025\"\n");
return 1; return 1;
} }
printf("SUCCESS\n"); printf("SUCCESS\n\n");
}
printf("Stress with random payloads of different encoding:\n");
{
int i, idx, where, len;
long long v;
unsigned char *p;
char buf[0x4041]; /* max length of generated string */
zl = ziplistNew();
for (i = 0; i < 100000; i++) {
where = (rand() & 1) ? ZIPLIST_HEAD : ZIPLIST_TAIL;
if (rand() & 1) {
/* equally likely create a 16, 32 or 64 bit int */
v = (rand() & INT16_MAX) + ((1ll << 32) >> ((rand() % 3)*16));
v *= 2*(rand() & 1)-1; /* randomly flip sign */
sprintf(buf, "%lld", v);
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), where);
} else {
/* equally likely generate 6, 14 or >14 bit length */
v = rand() & 0x3f;
v += 0x4000 >> ((rand() % 3)*8);
memset(buf, 'x', v);
zl = ziplistPush(zl, (unsigned char*)buf, v, where);
}
/* delete a random element */
if ((len = ziplistLen(zl)) >= 10) {
idx = rand() % len;
// printf("Delete index %d\n", idx);
// ziplistRepr(zl);
ziplistDeleteRange(zl, idx, 1);
// ziplistRepr(zl);
len--;
}
/* iterate from front to back */
idx = 0;
p = ziplistIndex(zl, 0);
while((p = ziplistNext(zl,p)))
idx++;
assert(len == idx+1);
/* iterate from back to front */
idx = 0;
p = ziplistIndex(zl, -1);
while((p = ziplistPrev(zl,p)))
idx++;
assert(len == idx+1);
}
printf("SUCCESS\n\n");
} }
printf("Stress with variable ziplist size:\n"); printf("Stress with variable ziplist size:\n");
......
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <pthread.h> #include <pthread.h>
#include "config.h" #include "config.h"
#if defined(__sun) #if defined(__sun)
...@@ -170,3 +171,69 @@ size_t zmalloc_used_memory(void) { ...@@ -170,3 +171,69 @@ size_t zmalloc_used_memory(void) {
void zmalloc_enable_thread_safeness(void) { void zmalloc_enable_thread_safeness(void) {
zmalloc_thread_safe = 1; zmalloc_thread_safe = 1;
} }
/* Fragmentation = RSS / allocated-bytes */
#if defined(HAVE_PROCFS)
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
float zmalloc_get_fragmentation_ratio(void) {
size_t allocated = zmalloc_used_memory();
int page = sysconf(_SC_PAGESIZE);
size_t rss;
char buf[4096];
char filename[256];
int fd, count;
char *p, *x;
snprintf(filename,256,"/proc/%d/stat",getpid());
if ((fd = open(filename,O_RDONLY)) == -1) return 0;
if (read(fd,buf,4096) <= 0) {
close(fd);
return 0;
}
close(fd);
p = buf;
count = 23; /* RSS is the 24th field in /proc/<pid>/stat */
while(p && count--) {
p = strchr(p,' ');
if (p) p++;
}
if (!p) return 0;
x = strchr(p,' ');
if (!x) return 0;
*x = '\0';
rss = strtoll(p,NULL,10);
rss *= page;
return (float)rss/allocated;
}
#elif defined(HAVE_TASKINFO)
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/sysctl.h>
#include <mach/task.h>
#include <mach/mach_init.h>
float zmalloc_get_fragmentation_ratio(void) {
task_t task = MACH_PORT_NULL;
struct task_basic_info t_info;
mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
if (task_for_pid(current_task(), getpid(), &task) != KERN_SUCCESS)
return 0;
task_info(task, TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count);
return (float)t_info.resident_size/zmalloc_used_memory();
}
#else
float zmalloc_get_fragmentation_ratio(void) {
return 0;
}
#endif
...@@ -38,5 +38,6 @@ void zfree(void *ptr); ...@@ -38,5 +38,6 @@ void zfree(void *ptr);
char *zstrdup(const char *s); char *zstrdup(const char *s);
size_t zmalloc_used_memory(void); size_t zmalloc_used_memory(void);
void zmalloc_enable_thread_safeness(void); void zmalloc_enable_thread_safeness(void);
float zmalloc_get_fragmentation_ratio(void);
#endif /* _ZMALLOC_H */ #endif /* _ZMALLOC_H */
start_server {tags {"cli"}} {
proc open_cli {} {
set ::env(TERM) dumb
set fd [open [format "|src/redis-cli -p %d -n 9" [srv port]] "r+"]
fconfigure $fd -buffering none
fconfigure $fd -blocking false
fconfigure $fd -translation binary
assert_equal "redis> " [read_cli $fd]
set _ $fd
}
proc close_cli {fd} {
close $fd
}
proc read_cli {fd} {
set buf [read $fd]
while {[string length $buf] == 0} {
# wait some time and try again
after 10
set buf [read $fd]
}
set _ $buf
}
proc write_cli {fd buf} {
puts $fd $buf
flush $fd
}
# Helpers to run tests in interactive mode
proc run_command {fd cmd} {
write_cli $fd $cmd
set lines [split [read_cli $fd] "\n"]
assert_equal "redis> " [lindex $lines end]
join [lrange $lines 0 end-1] "\n"
}
proc test_interactive_cli {name code} {
set ::env(FAKETTY) 1
set fd [open_cli]
test "Interactive CLI: $name" $code
close_cli $fd
unset ::env(FAKETTY)
}
# Helpers to run tests where stdout is not a tty
proc write_tmpfile {contents} {
set tmp [tmpfile "cli"]
set tmpfd [open $tmp "w"]
puts -nonewline $tmpfd $contents
close $tmpfd
set _ $tmp
}
proc _run_cli {opts args} {
set cmd [format "src/redis-cli -p %d -n 9 $args" [srv port]]
foreach {key value} $opts {
if {$key eq "pipe"} {
set cmd "sh -c \"$value | $cmd\""
}
if {$key eq "path"} {
set cmd "$cmd < $value"
}
}
set fd [open "|$cmd" "r"]
fconfigure $fd -buffering none
fconfigure $fd -translation binary
set resp [read $fd 1048576]
close $fd
set _ $resp
}
proc run_cli {args} {
_run_cli {} {*}$args
}
proc run_cli_with_input_pipe {cmd args} {
_run_cli [list pipe $cmd] {*}$args
}
proc run_cli_with_input_file {path args} {
_run_cli [list path $path] {*}$args
}
proc test_nontty_cli {name code} {
test "Non-interactive non-TTY CLI: $name" $code
}
# Helpers to run tests where stdout is a tty (fake it)
proc test_tty_cli {name code} {
set ::env(FAKETTY) 1
test "Non-interactive TTY CLI: $name" $code
unset ::env(FAKETTY)
}
test_interactive_cli "INFO response should be printed raw" {
set lines [split [run_command $fd info] "\n"]
foreach line $lines {
assert [regexp {^[a-z0-9_]+:[a-z0-9_]+} $line]
}
}
test_interactive_cli "Status reply" {
assert_equal "OK" [run_command $fd "set key foo"]
}
test_interactive_cli "Integer reply" {
assert_equal "(integer) 1" [run_command $fd "incr counter"]
}
test_interactive_cli "Bulk reply" {
r set key foo
assert_equal "\"foo\"" [run_command $fd "get key"]
}
test_interactive_cli "Multi-bulk reply" {
r rpush list foo
r rpush list bar
assert_equal "1. \"foo\"\n2. \"bar\"" [run_command $fd "lrange list 0 -1"]
}
test_interactive_cli "Parsing quotes" {
assert_equal "OK" [run_command $fd "set key \"bar\""]
assert_equal "bar" [r get key]
assert_equal "OK" [run_command $fd "set key \" bar \""]
assert_equal " bar " [r get key]
assert_equal "OK" [run_command $fd "set key \"\\\"bar\\\"\""]
assert_equal "\"bar\"" [r get key]
assert_equal "OK" [run_command $fd "set key \"\tbar\t\""]
assert_equal "\tbar\t" [r get key]
# invalid quotation
assert_equal "Invalid argument(s)" [run_command $fd "get \"\"key"]
assert_equal "Invalid argument(s)" [run_command $fd "get \"key\"x"]
# quotes after the argument are weird, but should be allowed
assert_equal "OK" [run_command $fd "set key\"\" bar"]
assert_equal "bar" [r get key]
}
test_tty_cli "Status reply" {
assert_equal "OK\n" [run_cli set key bar]
assert_equal "bar" [r get key]
}
test_tty_cli "Integer reply" {
r del counter
assert_equal "(integer) 1\n" [run_cli incr counter]
}
test_tty_cli "Bulk reply" {
r set key "tab\tnewline\n"
assert_equal "\"tab\\tnewline\\n\"\n" [run_cli get key]
}
test_tty_cli "Multi-bulk reply" {
r del list
r rpush list foo
r rpush list bar
assert_equal "1. \"foo\"\n2. \"bar\"\n" [run_cli lrange list 0 -1]
}
test_tty_cli "Read last argument from pipe" {
assert_equal "OK\n" [run_cli_with_input_pipe "echo foo" set key]
assert_equal "foo\n" [r get key]
}
test_tty_cli "Read last argument from file" {
set tmpfile [write_tmpfile "from file"]
assert_equal "OK\n" [run_cli_with_input_file $tmpfile set key]
assert_equal "from file" [r get key]
}
test_nontty_cli "Status reply" {
assert_equal "OK" [run_cli set key bar]
assert_equal "bar" [r get key]
}
test_nontty_cli "Integer reply" {
r del counter
assert_equal "1" [run_cli incr counter]
}
test_nontty_cli "Bulk reply" {
r set key "tab\tnewline\n"
assert_equal "tab\tnewline\n" [run_cli get key]
}
test_nontty_cli "Multi-bulk reply" {
r del list
r rpush list foo
r rpush list bar
assert_equal "foo\nbar" [run_cli lrange list 0 -1]
}
test_nontty_cli "Read last argument from pipe" {
assert_equal "OK" [run_cli_with_input_pipe "echo foo" set key]
assert_equal "foo\n" [r get key]
}
test_nontty_cli "Read last argument from file" {
set tmpfile [write_tmpfile "from file"]
assert_equal "OK" [run_cli_with_input_file $tmpfile set key]
assert_equal "from file" [r get key]
}
}
...@@ -23,6 +23,24 @@ start_server {tags {"repl"}} { ...@@ -23,6 +23,24 @@ start_server {tags {"repl"}} {
} }
assert_equal [r debug digest] [r -1 debug digest] assert_equal [r debug digest] [r -1 debug digest]
} }
test {MASTER and SLAVE consistency with expire} {
createComplexDataset r 50000 useexpire
after 4000 ;# Make sure everything expired before taking the digest
if {[r debug digest] ne [r -1 debug digest]} {
set csv1 [csvdump r]
set csv2 [csvdump {r -1}]
set fd [open /tmp/repldump1.txt w]
puts -nonewline $fd $csv1
close $fd
set fd [open /tmp/repldump2.txt w]
puts -nonewline $fd $csv2
close $fd
puts "Master - Slave inconsistency"
puts "Run diff -u against /tmp/repldump*.txt for more info"
}
assert_equal [r debug digest] [r -1 debug digest]
}
} }
} }
......
...@@ -83,7 +83,9 @@ proc ping_server {host port} { ...@@ -83,7 +83,9 @@ proc ping_server {host port} {
} }
close $fd close $fd
} e]} { } e]} {
puts "Can't PING server at $host:$port... $e" puts -nonewline "."
} else {
puts -nonewline "ok"
} }
return $retval return $retval
} }
...@@ -170,14 +172,33 @@ proc start_server {options {code undefined}} { ...@@ -170,14 +172,33 @@ proc start_server {options {code undefined}} {
if {$::valgrind} { if {$::valgrind} {
exec valgrind src/redis-server $config_file > $stdout 2> $stderr & exec valgrind src/redis-server $config_file > $stdout 2> $stderr &
after 2000
} else { } else {
exec src/redis-server $config_file > $stdout 2> $stderr & exec src/redis-server $config_file > $stdout 2> $stderr &
after 500
} }
# check that the server actually started # check that the server actually started
if {$code ne "undefined" && ![ping_server $::host $::port]} { # ugly but tries to be as fast as possible...
set retrynum 20
set serverisup 0
puts -nonewline "=== ($tags) Starting server ${::host}:${::port} "
after 10
if {$code ne "undefined"} {
while {[incr retrynum -1]} {
catch {
if {[ping_server $::host $::port]} {
set serverisup 1
}
}
if {$serverisup} break
after 50
}
} else {
set serverisup 1
}
puts {}
if {!$serverisup} {
error_and_quit $config_file [exec cat $stderr] error_and_quit $config_file [exec cat $stderr]
} }
...@@ -230,7 +251,11 @@ proc start_server {options {code undefined}} { ...@@ -230,7 +251,11 @@ proc start_server {options {code undefined}} {
# execute provided block # execute provided block
set curnum $::testnum set curnum $::testnum
catch { uplevel 1 $code } err if {![catch { uplevel 1 $code } err]} {
# zero exit status is good
unset err
}
if {$curnum == $::testnum} { if {$curnum == $::testnum} {
# don't check for leaks when no tests were executed # don't check for leaks when no tests were executed
dict set srv "skipleaks" 1 dict set srv "skipleaks" 1
...@@ -241,6 +266,7 @@ proc start_server {options {code undefined}} { ...@@ -241,6 +266,7 @@ proc start_server {options {code undefined}} {
# allow an exception to bubble up the call chain but still kill this # allow an exception to bubble up the call chain but still kill this
# server, because we want to reuse the ports when the tests are re-run # server, because we want to reuse the ports when the tests are re-run
if {[info exists err]} {
if {$err eq "exception"} { if {$err eq "exception"} {
puts [format "Logged warnings (pid %d):" [dict get $srv "pid"]] puts [format "Logged warnings (pid %d):" [dict get $srv "pid"]]
set warnings [warnings_from_file [dict get $srv "stdout"]] set warnings [warnings_from_file [dict get $srv "stdout"]]
...@@ -258,6 +284,7 @@ proc start_server {options {code undefined}} { ...@@ -258,6 +284,7 @@ proc start_server {options {code undefined}} {
puts $err puts $err
exit 1 exit 1
} }
}
set ::tags [lrange $::tags 0 end-[llength $tags]] set ::tags [lrange $::tags 0 end-[llength $tags]]
kill_server $srv kill_server $srv
......
...@@ -36,8 +36,8 @@ proc assert_encoding {enc key} { ...@@ -36,8 +36,8 @@ proc assert_encoding {enc key} {
# Swapped out values don't have an encoding, so make sure that # Swapped out values don't have an encoding, so make sure that
# the value is swapped in before checking the encoding. # the value is swapped in before checking the encoding.
set dbg [r debug object $key] set dbg [r debug object $key]
while {[string match "* swapped:*" $dbg]} { while {[string match "* swapped at:*" $dbg]} {
[r debug swapin $key] r debug swapin $key
set dbg [r debug object $key] set dbg [r debug object $key]
} }
assert_match "* encoding:$enc *" $dbg assert_match "* encoding:$enc *" $dbg
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment