Unverified Commit 5cbc73c6 authored by jared's avatar jared Committed by GitHub
Browse files

Merge pull request #1 from antirez/unstable

merge redis
parents a16aa03a d5536e04
...@@ -142,11 +142,11 @@ void slowlogReset(void) { ...@@ -142,11 +142,11 @@ void slowlogReset(void) {
void slowlogCommand(client *c) { void slowlogCommand(client *c) {
if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
const char *help[] = { const char *help[] = {
"get [count] -- Return top entries from the slowlog (default: 10)." "GET [count] -- Return top entries from the slowlog (default: 10)."
" Entries are made of:", " Entries are made of:",
" id, timestamp, time in microseconds, arguments array, client IP and port, client name", " id, timestamp, time in microseconds, arguments array, client IP and port, client name",
"len -- Return the length of the slowlog.", "LEN -- Return the length of the slowlog.",
"reset -- Reset the slowlog.", "RESET -- Reset the slowlog.",
NULL NULL
}; };
addReplyHelp(c, help); addReplyHelp(c, help);
...@@ -169,24 +169,24 @@ NULL ...@@ -169,24 +169,24 @@ NULL
return; return;
listRewind(server.slowlog,&li); listRewind(server.slowlog,&li);
totentries = addDeferredMultiBulkLength(c); totentries = addReplyDeferredLen(c);
while(count-- && (ln = listNext(&li))) { while(count-- && (ln = listNext(&li))) {
int j; int j;
se = ln->value; se = ln->value;
addReplyMultiBulkLen(c,6); addReplyArrayLen(c,6);
addReplyLongLong(c,se->id); addReplyLongLong(c,se->id);
addReplyLongLong(c,se->time); addReplyLongLong(c,se->time);
addReplyLongLong(c,se->duration); addReplyLongLong(c,se->duration);
addReplyMultiBulkLen(c,se->argc); addReplyArrayLen(c,se->argc);
for (j = 0; j < se->argc; j++) for (j = 0; j < se->argc; j++)
addReplyBulk(c,se->argv[j]); addReplyBulk(c,se->argv[j]);
addReplyBulkCBuffer(c,se->peerid,sdslen(se->peerid)); addReplyBulkCBuffer(c,se->peerid,sdslen(se->peerid));
addReplyBulkCBuffer(c,se->cname,sdslen(se->cname)); addReplyBulkCBuffer(c,se->cname,sdslen(se->cname));
sent++; sent++;
} }
setDeferredMultiBulkLength(c,totentries,sent); setDeferredArrayLen(c,totentries,sent);
} else { } else {
addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try SLOWLOG HELP", (char*)c->argv[1]->ptr); addReplySubcommandSyntaxError(c);
} }
} }
...@@ -58,7 +58,7 @@ redisSortOperation *createSortOperation(int type, robj *pattern) { ...@@ -58,7 +58,7 @@ redisSortOperation *createSortOperation(int type, robj *pattern) {
* *
* The returned object will always have its refcount increased by 1 * The returned object will always have its refcount increased by 1
* when it is non-NULL. */ * when it is non-NULL. */
robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) { robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst, int writeflag) {
char *p, *f, *k; char *p, *f, *k;
sds spat, ssub; sds spat, ssub;
robj *keyobj, *fieldobj = NULL, *o; robj *keyobj, *fieldobj = NULL, *o;
...@@ -106,7 +106,10 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) { ...@@ -106,7 +106,10 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) {
decrRefCount(subst); /* Incremented by decodeObject() */ decrRefCount(subst); /* Incremented by decodeObject() */
/* Lookup substituted key */ /* Lookup substituted key */
if (!writeflag)
o = lookupKeyRead(db,keyobj); o = lookupKeyRead(db,keyobj);
else
o = lookupKeyWrite(db,keyobj);
if (o == NULL) goto noobj; if (o == NULL) goto noobj;
if (fieldobj) { if (fieldobj) {
...@@ -193,35 +196,17 @@ void sortCommand(client *c) { ...@@ -193,35 +196,17 @@ void sortCommand(client *c) {
long limit_start = 0, limit_count = -1, start, end; long limit_start = 0, limit_count = -1, start, end;
int j, dontsort = 0, vectorlen; int j, dontsort = 0, vectorlen;
int getop = 0; /* GET operation counter */ int getop = 0; /* GET operation counter */
int int_convertion_error = 0; int int_conversion_error = 0;
int syntax_error = 0; int syntax_error = 0;
robj *sortval, *sortby = NULL, *storekey = NULL; robj *sortval, *sortby = NULL, *storekey = NULL;
redisSortObject *vector; /* Resulting vector to sort */ redisSortObject *vector; /* Resulting vector to sort */
/* Lookup the key to sort. It must be of the right types */
sortval = lookupKeyRead(c->db,c->argv[1]);
if (sortval && sortval->type != OBJ_SET &&
sortval->type != OBJ_LIST &&
sortval->type != OBJ_ZSET)
{
addReply(c,shared.wrongtypeerr);
return;
}
/* Create a list of operations to perform for every sorted element. /* Create a list of operations to perform for every sorted element.
* Operations can be GET */ * Operations can be GET */
operations = listCreate(); operations = listCreate();
listSetFreeMethod(operations,zfree); listSetFreeMethod(operations,zfree);
j = 2; /* options start at argv[2] */ j = 2; /* options start at argv[2] */
/* Now we need to protect sortval incrementing its count, in the future
* SORT may have options able to overwrite/delete keys during the sorting
* and the sorted key itself may get destroyed */
if (sortval)
incrRefCount(sortval);
else
sortval = createQuicklistObject();
/* The SORT command has an SQL-alike syntax, parse it */ /* The SORT command has an SQL-alike syntax, parse it */
while(j < c->argc) { while(j < c->argc) {
int leftargs = c->argc-j-1; int leftargs = c->argc-j-1;
...@@ -280,11 +265,33 @@ void sortCommand(client *c) { ...@@ -280,11 +265,33 @@ void sortCommand(client *c) {
/* Handle syntax errors set during options parsing. */ /* Handle syntax errors set during options parsing. */
if (syntax_error) { if (syntax_error) {
decrRefCount(sortval);
listRelease(operations); listRelease(operations);
return; return;
} }
/* Lookup the key to sort. It must be of the right types */
if (storekey)
sortval = lookupKeyRead(c->db,c->argv[1]);
else
sortval = lookupKeyWrite(c->db,c->argv[1]);
if (sortval && sortval->type != OBJ_SET &&
sortval->type != OBJ_LIST &&
sortval->type != OBJ_ZSET)
{
listRelease(operations);
addReply(c,shared.wrongtypeerr);
return;
}
/* Now we need to protect sortval incrementing its count, in the future
* SORT may have options able to overwrite/delete keys during the sorting
* and the sorted key itself may get destroyed */
if (sortval)
incrRefCount(sortval);
else
sortval = createQuicklistObject();
/* When sorting a set with no sort specified, we must sort the output /* When sorting a set with no sort specified, we must sort the output
* so the result is consistent across scripting and replication. * so the result is consistent across scripting and replication.
* *
...@@ -447,12 +454,12 @@ void sortCommand(client *c) { ...@@ -447,12 +454,12 @@ void sortCommand(client *c) {
serverAssertWithInfo(c,sortval,j == vectorlen); serverAssertWithInfo(c,sortval,j == vectorlen);
/* Now it's time to load the right scores in the sorting vector */ /* Now it's time to load the right scores in the sorting vector */
if (dontsort == 0) { if (!dontsort) {
for (j = 0; j < vectorlen; j++) { for (j = 0; j < vectorlen; j++) {
robj *byval; robj *byval;
if (sortby) { if (sortby) {
/* lookup value to sort by */ /* lookup value to sort by */
byval = lookupKeyByPattern(c->db,sortby,vector[j].obj); byval = lookupKeyByPattern(c->db,sortby,vector[j].obj,storekey!=NULL);
if (!byval) continue; if (!byval) continue;
} else { } else {
/* use object itself to sort by */ /* use object itself to sort by */
...@@ -469,7 +476,7 @@ void sortCommand(client *c) { ...@@ -469,7 +476,7 @@ void sortCommand(client *c) {
if (eptr[0] != '\0' || errno == ERANGE || if (eptr[0] != '\0' || errno == ERANGE ||
isnan(vector[j].u.score)) isnan(vector[j].u.score))
{ {
int_convertion_error = 1; int_conversion_error = 1;
} }
} else if (byval->encoding == OBJ_ENCODING_INT) { } else if (byval->encoding == OBJ_ENCODING_INT) {
/* Don't need to decode the object if it's /* Don't need to decode the object if it's
...@@ -487,9 +494,7 @@ void sortCommand(client *c) { ...@@ -487,9 +494,7 @@ void sortCommand(client *c) {
decrRefCount(byval); decrRefCount(byval);
} }
} }
}
if (dontsort == 0) {
server.sort_desc = desc; server.sort_desc = desc;
server.sort_alpha = alpha; server.sort_alpha = alpha;
server.sort_bypattern = sortby ? 1 : 0; server.sort_bypattern = sortby ? 1 : 0;
...@@ -503,11 +508,11 @@ void sortCommand(client *c) { ...@@ -503,11 +508,11 @@ void sortCommand(client *c) {
/* Send command output to the output buffer, performing the specified /* Send command output to the output buffer, performing the specified
* GET/DEL/INCR/DECR operations if any. */ * GET/DEL/INCR/DECR operations if any. */
outputlen = getop ? getop*(end-start+1) : end-start+1; outputlen = getop ? getop*(end-start+1) : end-start+1;
if (int_convertion_error) { if (int_conversion_error) {
addReplyError(c,"One or more scores can't be converted into double"); addReplyError(c,"One or more scores can't be converted into double");
} else if (storekey == NULL) { } else if (storekey == NULL) {
/* STORE option not specified, sent the sorting result to client */ /* STORE option not specified, sent the sorting result to client */
addReplyMultiBulkLen(c,outputlen); addReplyArrayLen(c,outputlen);
for (j = start; j <= end; j++) { for (j = start; j <= end; j++) {
listNode *ln; listNode *ln;
listIter li; listIter li;
...@@ -517,11 +522,11 @@ void sortCommand(client *c) { ...@@ -517,11 +522,11 @@ void sortCommand(client *c) {
while((ln = listNext(&li))) { while((ln = listNext(&li))) {
redisSortOperation *sop = ln->value; redisSortOperation *sop = ln->value;
robj *val = lookupKeyByPattern(c->db,sop->pattern, robj *val = lookupKeyByPattern(c->db,sop->pattern,
vector[j].obj); vector[j].obj,storekey!=NULL);
if (sop->type == SORT_OP_GET) { if (sop->type == SORT_OP_GET) {
if (!val) { if (!val) {
addReply(c,shared.nullbulk); addReplyNull(c);
} else { } else {
addReplyBulk(c,val); addReplyBulk(c,val);
decrRefCount(val); decrRefCount(val);
...@@ -547,7 +552,7 @@ void sortCommand(client *c) { ...@@ -547,7 +552,7 @@ void sortCommand(client *c) {
while((ln = listNext(&li))) { while((ln = listNext(&li))) {
redisSortOperation *sop = ln->value; redisSortOperation *sop = ln->value;
robj *val = lookupKeyByPattern(c->db,sop->pattern, robj *val = lookupKeyByPattern(c->db,sop->pattern,
vector[j].obj); vector[j].obj,storekey!=NULL);
if (sop->type == SORT_OP_GET) { if (sop->type == SORT_OP_GET) {
if (!val) val = createStringObject("",0); if (!val) val = createStringObject("",0);
......
...@@ -108,5 +108,7 @@ streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create); ...@@ -108,5 +108,7 @@ streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create);
streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id); streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id);
streamNACK *streamCreateNACK(streamConsumer *consumer); streamNACK *streamCreateNACK(streamConsumer *consumer);
void streamDecodeID(void *buf, streamID *id); void streamDecodeID(void *buf, streamID *id);
int streamCompareID(streamID *a, streamID *b);
void streamFreeNACK(streamNACK *na);
#endif #endif
...@@ -615,6 +615,10 @@ void hincrbyfloatCommand(client *c) { ...@@ -615,6 +615,10 @@ void hincrbyfloatCommand(client *c) {
} }
value += incr; value += incr;
if (isnan(value) || isinf(value)) {
addReplyError(c,"increment would produce NaN or Infinity");
return;
}
char buf[MAX_LONG_DOUBLE_CHARS]; char buf[MAX_LONG_DOUBLE_CHARS];
int len = ld2string(buf,sizeof(buf),value,1); int len = ld2string(buf,sizeof(buf),value,1);
...@@ -641,7 +645,7 @@ static void addHashFieldToReply(client *c, robj *o, sds field) { ...@@ -641,7 +645,7 @@ static void addHashFieldToReply(client *c, robj *o, sds field) {
int ret; int ret;
if (o == NULL) { if (o == NULL) {
addReply(c, shared.nullbulk); addReplyNull(c);
return; return;
} }
...@@ -652,7 +656,7 @@ static void addHashFieldToReply(client *c, robj *o, sds field) { ...@@ -652,7 +656,7 @@ static void addHashFieldToReply(client *c, robj *o, sds field) {
ret = hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll); ret = hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll);
if (ret < 0) { if (ret < 0) {
addReply(c, shared.nullbulk); addReplyNull(c);
} else { } else {
if (vstr) { if (vstr) {
addReplyBulkCBuffer(c, vstr, vlen); addReplyBulkCBuffer(c, vstr, vlen);
...@@ -664,7 +668,7 @@ static void addHashFieldToReply(client *c, robj *o, sds field) { ...@@ -664,7 +668,7 @@ static void addHashFieldToReply(client *c, robj *o, sds field) {
} else if (o->encoding == OBJ_ENCODING_HT) { } else if (o->encoding == OBJ_ENCODING_HT) {
sds value = hashTypeGetFromHashTable(o, field); sds value = hashTypeGetFromHashTable(o, field);
if (value == NULL) if (value == NULL)
addReply(c, shared.nullbulk); addReplyNull(c);
else else
addReplyBulkCBuffer(c, value, sdslen(value)); addReplyBulkCBuffer(c, value, sdslen(value));
} else { } else {
...@@ -675,7 +679,7 @@ static void addHashFieldToReply(client *c, robj *o, sds field) { ...@@ -675,7 +679,7 @@ static void addHashFieldToReply(client *c, robj *o, sds field) {
void hgetCommand(client *c) { void hgetCommand(client *c) {
robj *o; robj *o;
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk)) == NULL || if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL ||
checkType(c,o,OBJ_HASH)) return; checkType(c,o,OBJ_HASH)) return;
addHashFieldToReply(c, o, c->argv[2]->ptr); addHashFieldToReply(c, o, c->argv[2]->ptr);
...@@ -693,7 +697,7 @@ void hmgetCommand(client *c) { ...@@ -693,7 +697,7 @@ void hmgetCommand(client *c) {
return; return;
} }
addReplyMultiBulkLen(c, c->argc-2); addReplyArrayLen(c, c->argc-2);
for (i = 2; i < c->argc; i++) { for (i = 2; i < c->argc; i++) {
addHashFieldToReply(c, o, c->argv[i]->ptr); addHashFieldToReply(c, o, c->argv[i]->ptr);
} }
...@@ -766,17 +770,19 @@ static void addHashIteratorCursorToReply(client *c, hashTypeIterator *hi, int wh ...@@ -766,17 +770,19 @@ static void addHashIteratorCursorToReply(client *c, hashTypeIterator *hi, int wh
void genericHgetallCommand(client *c, int flags) { void genericHgetallCommand(client *c, int flags) {
robj *o; robj *o;
hashTypeIterator *hi; hashTypeIterator *hi;
int multiplier = 0;
int length, count = 0; int length, count = 0;
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) == NULL if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL
|| checkType(c,o,OBJ_HASH)) return; || checkType(c,o,OBJ_HASH)) return;
if (flags & OBJ_HASH_KEY) multiplier++; /* We return a map if the user requested keys and values, like in the
if (flags & OBJ_HASH_VALUE) multiplier++; * HGETALL case. Otherwise to use a flat array makes more sense. */
length = hashTypeLength(o);
length = hashTypeLength(o) * multiplier; if (flags & OBJ_HASH_KEY && flags & OBJ_HASH_VALUE) {
addReplyMultiBulkLen(c, length); addReplyMapLen(c, length);
} else {
addReplyArrayLen(c, length);
}
hi = hashTypeInitIterator(o); hi = hashTypeInitIterator(o);
while (hashTypeNext(hi) != C_ERR) { while (hashTypeNext(hi) != C_ERR) {
...@@ -791,6 +797,9 @@ void genericHgetallCommand(client *c, int flags) { ...@@ -791,6 +797,9 @@ void genericHgetallCommand(client *c, int flags) {
} }
hashTypeReleaseIterator(hi); hashTypeReleaseIterator(hi);
/* Make sure we returned the right number of elements. */
if (flags & OBJ_HASH_KEY && flags & OBJ_HASH_VALUE) count /= 2;
serverAssert(count == length); serverAssert(count == length);
} }
......
...@@ -298,7 +298,7 @@ void linsertCommand(client *c) { ...@@ -298,7 +298,7 @@ void linsertCommand(client *c) {
server.dirty++; server.dirty++;
} else { } else {
/* Notify client of a failed insert */ /* Notify client of a failed insert */
addReply(c,shared.cnegone); addReplyLongLong(c,-1);
return; return;
} }
...@@ -312,7 +312,7 @@ void llenCommand(client *c) { ...@@ -312,7 +312,7 @@ void llenCommand(client *c) {
} }
void lindexCommand(client *c) { void lindexCommand(client *c) {
robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk); robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]);
if (o == NULL || checkType(c,o,OBJ_LIST)) return; if (o == NULL || checkType(c,o,OBJ_LIST)) return;
long index; long index;
robj *value = NULL; robj *value = NULL;
...@@ -331,7 +331,7 @@ void lindexCommand(client *c) { ...@@ -331,7 +331,7 @@ void lindexCommand(client *c) {
addReplyBulk(c,value); addReplyBulk(c,value);
decrRefCount(value); decrRefCount(value);
} else { } else {
addReply(c,shared.nullbulk); addReplyNull(c);
} }
} else { } else {
serverPanic("Unknown list encoding"); serverPanic("Unknown list encoding");
...@@ -365,12 +365,12 @@ void lsetCommand(client *c) { ...@@ -365,12 +365,12 @@ void lsetCommand(client *c) {
} }
void popGenericCommand(client *c, int where) { void popGenericCommand(client *c, int where) {
robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nullbulk); robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]);
if (o == NULL || checkType(c,o,OBJ_LIST)) return; if (o == NULL || checkType(c,o,OBJ_LIST)) return;
robj *value = listTypePop(o,where); robj *value = listTypePop(o,where);
if (value == NULL) { if (value == NULL) {
addReply(c,shared.nullbulk); addReplyNull(c);
} else { } else {
char *event = (where == LIST_HEAD) ? "lpop" : "rpop"; char *event = (where == LIST_HEAD) ? "lpop" : "rpop";
...@@ -402,7 +402,7 @@ void lrangeCommand(client *c) { ...@@ -402,7 +402,7 @@ void lrangeCommand(client *c) {
if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) || if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) ||
(getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return; (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return;
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) == NULL if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL
|| checkType(c,o,OBJ_LIST)) return; || checkType(c,o,OBJ_LIST)) return;
llen = listTypeLength(o); llen = listTypeLength(o);
...@@ -414,14 +414,14 @@ void lrangeCommand(client *c) { ...@@ -414,14 +414,14 @@ void lrangeCommand(client *c) {
/* Invariant: start >= 0, so this test will be true when end < 0. /* Invariant: start >= 0, so this test will be true when end < 0.
* The range is empty when start > end or start >= length. */ * The range is empty when start > end or start >= length. */
if (start > end || start >= llen) { if (start > end || start >= llen) {
addReply(c,shared.emptymultibulk); addReplyNull(c);
return; return;
} }
if (end >= llen) end = llen-1; if (end >= llen) end = llen-1;
rangelen = (end-start)+1; rangelen = (end-start)+1;
/* Return the result in form of a multi-bulk reply */ /* Return the result in form of a multi-bulk reply */
addReplyMultiBulkLen(c,rangelen); addReplyArrayLen(c,rangelen);
if (o->encoding == OBJ_ENCODING_QUICKLIST) { if (o->encoding == OBJ_ENCODING_QUICKLIST) {
listTypeIterator *iter = listTypeInitIterator(o, start, LIST_TAIL); listTypeIterator *iter = listTypeInitIterator(o, start, LIST_TAIL);
...@@ -520,7 +520,7 @@ void lremCommand(client *c) { ...@@ -520,7 +520,7 @@ void lremCommand(client *c) {
if (removed) { if (removed) {
signalModifiedKey(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_GENERIC,"lrem",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id);
} }
if (listTypeLength(subject) == 0) { if (listTypeLength(subject) == 0) {
...@@ -564,13 +564,13 @@ void rpoplpushHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value) { ...@@ -564,13 +564,13 @@ void rpoplpushHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value) {
void rpoplpushCommand(client *c) { void rpoplpushCommand(client *c) {
robj *sobj, *value; robj *sobj, *value;
if ((sobj = lookupKeyWriteOrReply(c,c->argv[1],shared.nullbulk)) == NULL || if ((sobj = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]))
checkType(c,sobj,OBJ_LIST)) return; == NULL || checkType(c,sobj,OBJ_LIST)) return;
if (listTypeLength(sobj) == 0) { if (listTypeLength(sobj) == 0) {
/* This may only happen after loading very old RDB files. Recent /* This may only happen after loading very old RDB files. Recent
* versions of Redis delete keys of empty lists. */ * versions of Redis delete keys of empty lists. */
addReply(c,shared.nullbulk); addReplyNull(c);
} else { } else {
robj *dobj = lookupKeyWrite(c->db,c->argv[2]); robj *dobj = lookupKeyWrite(c->db,c->argv[2]);
robj *touchedkey = c->argv[1]; robj *touchedkey = c->argv[1];
...@@ -596,6 +596,9 @@ void rpoplpushCommand(client *c) { ...@@ -596,6 +596,9 @@ void rpoplpushCommand(client *c) {
signalModifiedKey(c->db,touchedkey); signalModifiedKey(c->db,touchedkey);
decrRefCount(touchedkey); decrRefCount(touchedkey);
server.dirty++; server.dirty++;
if (c->cmd->proc == brpoplpushCommand) {
rewriteClientCommandVector(c,3,shared.rpoplpush,c->argv[1],c->argv[2]);
}
} }
} }
...@@ -614,7 +617,7 @@ void rpoplpushCommand(client *c) { ...@@ -614,7 +617,7 @@ void rpoplpushCommand(client *c) {
* the AOF and replication channel. * the AOF and replication channel.
* *
* The argument 'where' is LIST_TAIL or LIST_HEAD, and indicates if the * The argument 'where' is LIST_TAIL or LIST_HEAD, and indicates if the
* 'value' element was popped fron the head (BLPOP) or tail (BRPOP) so that * 'value' element was popped from the head (BLPOP) or tail (BRPOP) so that
* we can propagate the command properly. * we can propagate the command properly.
* *
* The function returns C_OK if we are able to serve the client, otherwise * The function returns C_OK if we are able to serve the client, otherwise
...@@ -636,7 +639,7 @@ int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb ...@@ -636,7 +639,7 @@ int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb
db->id,argv,2,PROPAGATE_AOF|PROPAGATE_REPL); db->id,argv,2,PROPAGATE_AOF|PROPAGATE_REPL);
/* BRPOP/BLPOP */ /* BRPOP/BLPOP */
addReplyMultiBulkLen(receiver,2); addReplyArrayLen(receiver,2);
addReplyBulk(receiver,key); addReplyBulk(receiver,key);
addReplyBulk(receiver,value); addReplyBulk(receiver,value);
...@@ -701,7 +704,7 @@ void blockingPopGenericCommand(client *c, int where) { ...@@ -701,7 +704,7 @@ void blockingPopGenericCommand(client *c, int where) {
robj *value = listTypePop(o,where); robj *value = listTypePop(o,where);
serverAssert(value != NULL); serverAssert(value != NULL);
addReplyMultiBulkLen(c,2); addReplyArrayLen(c,2);
addReplyBulk(c,c->argv[j]); addReplyBulk(c,c->argv[j]);
addReplyBulk(c,value); addReplyBulk(c,value);
decrRefCount(value); decrRefCount(value);
...@@ -728,7 +731,7 @@ void blockingPopGenericCommand(client *c, int where) { ...@@ -728,7 +731,7 @@ void blockingPopGenericCommand(client *c, int where) {
/* If we are inside a MULTI/EXEC and the list is empty the only thing /* If we are inside a MULTI/EXEC and the list is empty the only thing
* we can do is treating it as a timeout (even with timeout 0). */ * we can do is treating it as a timeout (even with timeout 0). */
if (c->flags & CLIENT_MULTI) { if (c->flags & CLIENT_MULTI) {
addReply(c,shared.nullmultibulk); addReplyNullArray(c);
return; return;
} }
...@@ -756,7 +759,7 @@ void brpoplpushCommand(client *c) { ...@@ -756,7 +759,7 @@ void brpoplpushCommand(client *c) {
if (c->flags & CLIENT_MULTI) { if (c->flags & CLIENT_MULTI) {
/* Blocking against an empty list in a multi state /* Blocking against an empty list in a multi state
* returns immediately. */ * returns immediately. */
addReply(c, shared.nullbulk); addReplyNull(c);
} else { } else {
/* The list is empty and the client blocks. */ /* The list is empty and the client blocks. */
blockForKeys(c,BLOCKED_LIST,c->argv + 1,1,timeout,c->argv[2],NULL); blockForKeys(c,BLOCKED_LIST,c->argv + 1,1,timeout,c->argv[2],NULL);
......
...@@ -207,7 +207,7 @@ sds setTypeNextObject(setTypeIterator *si) { ...@@ -207,7 +207,7 @@ sds setTypeNextObject(setTypeIterator *si) {
* used field with values which are easy to trap if misused. */ * used field with values which are easy to trap if misused. */
int setTypeRandomElement(robj *setobj, sds *sdsele, int64_t *llele) { int setTypeRandomElement(robj *setobj, sds *sdsele, int64_t *llele) {
if (setobj->encoding == OBJ_ENCODING_HT) { if (setobj->encoding == OBJ_ENCODING_HT) {
dictEntry *de = dictGetRandomKey(setobj->ptr); dictEntry *de = dictGetFairRandomKey(setobj->ptr);
*sdsele = dictGetKey(de); *sdsele = dictGetKey(de);
*llele = -123456789; /* Not needed. Defensive. */ *llele = -123456789; /* Not needed. Defensive. */
} else if (setobj->encoding == OBJ_ENCODING_INTSET) { } else if (setobj->encoding == OBJ_ENCODING_INTSET) {
...@@ -415,13 +415,13 @@ void spopWithCountCommand(client *c) { ...@@ -415,13 +415,13 @@ void spopWithCountCommand(client *c) {
/* Make sure a key with the name inputted exists, and that it's type is /* Make sure a key with the name inputted exists, and that it's type is
* indeed a set. Otherwise, return nil */ * indeed a set. Otherwise, return nil */
if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]))
== NULL || checkType(c,set,OBJ_SET)) return; == NULL || checkType(c,set,OBJ_SET)) return;
/* If count is zero, serve an empty multibulk ASAP to avoid special /* If count is zero, serve an empty multibulk ASAP to avoid special
* cases later. */ * cases later. */
if (count == 0) { if (count == 0) {
addReply(c,shared.emptymultibulk); addReplyNull(c);
return; return;
} }
...@@ -455,7 +455,7 @@ void spopWithCountCommand(client *c) { ...@@ -455,7 +455,7 @@ void spopWithCountCommand(client *c) {
robj *propargv[3]; robj *propargv[3];
propargv[0] = createStringObject("SREM",4); propargv[0] = createStringObject("SREM",4);
propargv[1] = c->argv[1]; propargv[1] = c->argv[1];
addReplyMultiBulkLen(c,count); addReplySetLen(c,count);
/* Common iteration vars. */ /* Common iteration vars. */
sds sdsele; sds sdsele;
...@@ -516,11 +516,7 @@ void spopWithCountCommand(client *c) { ...@@ -516,11 +516,7 @@ void spopWithCountCommand(client *c) {
sdsfree(sdsele); sdsfree(sdsele);
} }
/* Assign the new set as the key value. */ /* Transfer the old set to the client. */
incrRefCount(set); /* Protect the old set value. */
dbOverwrite(c->db,c->argv[1],newset);
/* Tranfer the old set to the client and release it. */
setTypeIterator *si; setTypeIterator *si;
si = setTypeInitIterator(set); si = setTypeInitIterator(set);
while((encoding = setTypeNext(si,&sdsele,&llele)) != -1) { while((encoding = setTypeNext(si,&sdsele,&llele)) != -1) {
...@@ -539,7 +535,9 @@ void spopWithCountCommand(client *c) { ...@@ -539,7 +535,9 @@ void spopWithCountCommand(client *c) {
decrRefCount(objele); decrRefCount(objele);
} }
setTypeReleaseIterator(si); setTypeReleaseIterator(si);
decrRefCount(set);
/* Assign the new set as the key value. */
dbOverwrite(c->db,c->argv[1],newset);
} }
/* Don't propagate the command itself even if we incremented the /* Don't propagate the command itself even if we incremented the
...@@ -568,8 +566,8 @@ void spopCommand(client *c) { ...@@ -568,8 +566,8 @@ void spopCommand(client *c) {
/* Make sure a key with the name inputted exists, and that it's type is /* Make sure a key with the name inputted exists, and that it's type is
* indeed a set */ * indeed a set */
if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.nullbulk)) == NULL || if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]))
checkType(c,set,OBJ_SET)) return; == NULL || checkType(c,set,OBJ_SET)) return;
/* Get a random element from the set */ /* Get a random element from the set */
encoding = setTypeRandomElement(set,&sdsele,&llele); encoding = setTypeRandomElement(set,&sdsele,&llele);
...@@ -634,13 +632,13 @@ void srandmemberWithCountCommand(client *c) { ...@@ -634,13 +632,13 @@ void srandmemberWithCountCommand(client *c) {
uniq = 0; uniq = 0;
} }
if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]))
== NULL || checkType(c,set,OBJ_SET)) return; == NULL || checkType(c,set,OBJ_SET)) return;
size = setTypeSize(set); size = setTypeSize(set);
/* If count is zero, serve it ASAP to avoid special cases later. */ /* If count is zero, serve it ASAP to avoid special cases later. */
if (count == 0) { if (count == 0) {
addReply(c,shared.emptymultibulk); addReplyNull(c);
return; return;
} }
...@@ -649,7 +647,7 @@ void srandmemberWithCountCommand(client *c) { ...@@ -649,7 +647,7 @@ void srandmemberWithCountCommand(client *c) {
* This case is trivial and can be served without auxiliary data * This case is trivial and can be served without auxiliary data
* structures. */ * structures. */
if (!uniq) { if (!uniq) {
addReplyMultiBulkLen(c,count); addReplySetLen(c,count);
while(count--) { while(count--) {
encoding = setTypeRandomElement(set,&ele,&llele); encoding = setTypeRandomElement(set,&ele,&llele);
if (encoding == OBJ_ENCODING_INTSET) { if (encoding == OBJ_ENCODING_INTSET) {
...@@ -739,7 +737,7 @@ void srandmemberWithCountCommand(client *c) { ...@@ -739,7 +737,7 @@ void srandmemberWithCountCommand(client *c) {
dictIterator *di; dictIterator *di;
dictEntry *de; dictEntry *de;
addReplyMultiBulkLen(c,count); addReplySetLen(c,count);
di = dictGetIterator(d); di = dictGetIterator(d);
while((de = dictNext(di)) != NULL) while((de = dictNext(di)) != NULL)
addReplyBulk(c,dictGetKey(de)); addReplyBulk(c,dictGetKey(de));
...@@ -762,8 +760,8 @@ void srandmemberCommand(client *c) { ...@@ -762,8 +760,8 @@ void srandmemberCommand(client *c) {
return; return;
} }
if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk)) == NULL || if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]))
checkType(c,set,OBJ_SET)) return; == NULL || checkType(c,set,OBJ_SET)) return;
encoding = setTypeRandomElement(set,&ele,&llele); encoding = setTypeRandomElement(set,&ele,&llele);
if (encoding == OBJ_ENCODING_INTSET) { if (encoding == OBJ_ENCODING_INTSET) {
...@@ -815,7 +813,7 @@ void sinterGenericCommand(client *c, robj **setkeys, ...@@ -815,7 +813,7 @@ void sinterGenericCommand(client *c, robj **setkeys,
} }
addReply(c,shared.czero); addReply(c,shared.czero);
} else { } else {
addReply(c,shared.emptymultibulk); addReplyNull(c);
} }
return; return;
} }
...@@ -835,7 +833,7 @@ void sinterGenericCommand(client *c, robj **setkeys, ...@@ -835,7 +833,7 @@ void sinterGenericCommand(client *c, robj **setkeys,
* to the output list and save the pointer to later modify it with the * to the output list and save the pointer to later modify it with the
* right length */ * right length */
if (!dstkey) { if (!dstkey) {
replylen = addDeferredMultiBulkLength(c); replylen = addReplyDeferredLen(c);
} else { } else {
/* If we have a target key where to store the resulting set /* If we have a target key where to store the resulting set
* create this key with an empty set inside */ * create this key with an empty set inside */
...@@ -913,7 +911,7 @@ void sinterGenericCommand(client *c, robj **setkeys, ...@@ -913,7 +911,7 @@ void sinterGenericCommand(client *c, robj **setkeys,
signalModifiedKey(c->db,dstkey); signalModifiedKey(c->db,dstkey);
server.dirty++; server.dirty++;
} else { } else {
setDeferredMultiBulkLength(c,replylen,cardinality); setDeferredSetLen(c,replylen,cardinality);
} }
zfree(sets); zfree(sets);
} }
...@@ -1059,13 +1057,14 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum, ...@@ -1059,13 +1057,14 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
/* Output the content of the resulting set, if not in STORE mode */ /* Output the content of the resulting set, if not in STORE mode */
if (!dstkey) { if (!dstkey) {
addReplyMultiBulkLen(c,cardinality); addReplySetLen(c,cardinality);
si = setTypeInitIterator(dstset); si = setTypeInitIterator(dstset);
while((ele = setTypeNextObject(si)) != NULL) { while((ele = setTypeNextObject(si)) != NULL) {
addReplyBulkCBuffer(c,ele,sdslen(ele)); addReplyBulkCBuffer(c,ele,sdslen(ele));
sdsfree(ele); sdsfree(ele);
} }
setTypeReleaseIterator(si); setTypeReleaseIterator(si);
server.lazyfree_lazy_server_del ? freeObjAsync(dstset) :
decrRefCount(dstset); decrRefCount(dstset);
} else { } else {
/* If we have a target key where to store the resulting set /* If we have a target key where to store the resulting set
......
...@@ -37,7 +37,7 @@ ...@@ -37,7 +37,7 @@
* mark the entry as deleted, or having the same field as the "master" * mark the entry as deleted, or having the same field as the "master"
* entry at the start of the listpack> */ * entry at the start of the listpack> */
#define STREAM_ITEM_FLAG_NONE 0 /* No special flags. */ #define STREAM_ITEM_FLAG_NONE 0 /* No special flags. */
#define STREAM_ITEM_FLAG_DELETED (1<<0) /* Entry is delted. Skip it. */ #define STREAM_ITEM_FLAG_DELETED (1<<0) /* Entry is deleted. Skip it. */
#define STREAM_ITEM_FLAG_SAMEFIELDS (1<<1) /* Same fields as master entry. */ #define STREAM_ITEM_FLAG_SAMEFIELDS (1<<1) /* Same fields as master entry. */
void streamFreeCG(streamCG *cg); void streamFreeCG(streamCG *cg);
...@@ -165,7 +165,7 @@ int streamCompareID(streamID *a, streamID *b) { ...@@ -165,7 +165,7 @@ int streamCompareID(streamID *a, streamID *b) {
* Returns the new entry ID populating the 'added_id' structure. * Returns the new entry ID populating the 'added_id' structure.
* *
* If 'use_id' is not NULL, the ID is not auto-generated by the function, * If 'use_id' is not NULL, the ID is not auto-generated by the function,
* but instead the passed ID is uesd to add the new entry. In this case * but instead the passed ID is used to add the new entry. In this case
* adding the entry may fail as specified later in this comment. * adding the entry may fail as specified later in this comment.
* *
* The function returns C_OK if the item was added, this is always true * The function returns C_OK if the item was added, this is always true
...@@ -207,7 +207,7 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ ...@@ -207,7 +207,7 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
/* Create a new listpack and radix tree node if needed. Note that when /* Create a new listpack and radix tree node if needed. Note that when
* a new listpack is created, we populate it with a "master entry". This * a new listpack is created, we populate it with a "master entry". This
* is just a set of fields that is taken as refernce in order to compress * is just a set of fields that is taken as references in order to compress
* the stream entries that we'll add inside the listpack. * the stream entries that we'll add inside the listpack.
* *
* Note that while we use the first added entry fields to create * Note that while we use the first added entry fields to create
...@@ -223,13 +223,13 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ ...@@ -223,13 +223,13 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
* *
* count and deleted just represent respectively the total number of * count and deleted just represent respectively the total number of
* entries inside the listpack that are valid, and marked as deleted * entries inside the listpack that are valid, and marked as deleted
* (delted flag in the entry flags set). So the total number of items * (deleted flag in the entry flags set). So the total number of items
* actually inside the listpack (both deleted and not) is count+deleted. * actually inside the listpack (both deleted and not) is count+deleted.
* *
* The real entries will be encoded with an ID that is just the * The real entries will be encoded with an ID that is just the
* millisecond and sequence difference compared to the key stored at * millisecond and sequence difference compared to the key stored at
* the radix tree node containing the listpack (delta encoding), and * the radix tree node containing the listpack (delta encoding), and
* if the fields of the entry are the same as the master enty fields, the * if the fields of the entry are the same as the master entry fields, the
* entry flags will specify this fact and the entry fields and number * entry flags will specify this fact and the entry fields and number
* of fields will be omitted (see later in the code of this function). * of fields will be omitted (see later in the code of this function).
* *
...@@ -349,6 +349,7 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ ...@@ -349,6 +349,7 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
lp = lpAppendInteger(lp,lp_count); lp = lpAppendInteger(lp,lp_count);
/* Insert back into the tree in order to update the listpack pointer. */ /* Insert back into the tree in order to update the listpack pointer. */
if (ri.data != lp)
raxInsert(s->rax,(unsigned char*)&rax_key,sizeof(rax_key),lp,NULL); raxInsert(s->rax,(unsigned char*)&rax_key,sizeof(rax_key),lp,NULL);
s->length++; s->length++;
s->last_id = id; s->last_id = id;
...@@ -469,7 +470,7 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) { ...@@ -469,7 +470,7 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) {
* iteration is from the start to the end element (inclusive), otherwise * iteration is from the start to the end element (inclusive), otherwise
* if rev is non-zero, the iteration is reversed. * if rev is non-zero, the iteration is reversed.
* *
* Once the iterator is initalized, we iterate like this: * Once the iterator is initialized, we iterate like this:
* *
* streamIterator myiterator; * streamIterator myiterator;
* streamIteratorStart(&myiterator,...); * streamIteratorStart(&myiterator,...);
...@@ -485,20 +486,20 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) { ...@@ -485,20 +486,20 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) {
* } * }
* streamIteratorStop(&myiterator); */ * streamIteratorStop(&myiterator); */
void streamIteratorStart(streamIterator *si, stream *s, streamID *start, streamID *end, int rev) { void streamIteratorStart(streamIterator *si, stream *s, streamID *start, streamID *end, int rev) {
/* Intialize the iterator and translates the iteration start/stop /* Initialize the iterator and translates the iteration start/stop
* elements into a 128 big big-endian number. */ * elements into a 128 big big-endian number. */
if (start) { if (start) {
streamEncodeID(si->start_key,start); streamEncodeID(si->start_key,start);
} else { } else {
si->start_key[0] = 0; si->start_key[0] = 0;
si->start_key[0] = 0; si->start_key[1] = 0;
} }
if (end) { if (end) {
streamEncodeID(si->end_key,end); streamEncodeID(si->end_key,end);
} else { } else {
si->end_key[0] = UINT64_MAX; si->end_key[0] = UINT64_MAX;
si->end_key[0] = UINT64_MAX; si->end_key[1] = UINT64_MAX;
} }
/* Seek the correct node in the radix tree. */ /* Seek the correct node in the radix tree. */
...@@ -548,15 +549,22 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { ...@@ -548,15 +549,22 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
si->master_fields_count = lpGetInteger(si->lp_ele); si->master_fields_count = lpGetInteger(si->lp_ele);
si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek first field. */ si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek first field. */
si->master_fields_start = si->lp_ele; si->master_fields_start = si->lp_ele;
/* Skip master fileds to seek the first entry. */ /* We are now pointing to the first field of the master entry.
* We need to seek either the first or the last entry depending
* on the direction of the iteration. */
if (!si->rev) {
/* If we are iterating in normal order, skip the master fields
* to seek the first actual entry. */
for (uint64_t i = 0; i < si->master_fields_count; i++) for (uint64_t i = 0; i < si->master_fields_count; i++)
si->lp_ele = lpNext(si->lp,si->lp_ele); si->lp_ele = lpNext(si->lp,si->lp_ele);
/* We are now pointing the zero term of the master entry. If } else {
* we are iterating in reverse order, we need to seek the /* If we are iterating in reverse direction, just seek the
* end of the listpack. */ * last part of the last entry in the listpack (that is, the
if (si->rev) si->lp_ele = lpLast(si->lp); * fields count). */
si->lp_ele = lpLast(si->lp);
}
} else if (si->rev) { } else if (si->rev) {
/* If we are itereating in the reverse order, and this is not /* If we are iterating in the reverse order, and this is not
* the first entry emitted for this listpack, then we already * the first entry emitted for this listpack, then we already
* emitted the current entry, and have to go back to the previous * emitted the current entry, and have to go back to the previous
* one. */ * one. */
...@@ -705,11 +713,24 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { ...@@ -705,11 +713,24 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) {
/* Change the valid/deleted entries count in the master entry. */ /* Change the valid/deleted entries count in the master entry. */
unsigned char *p = lpFirst(lp); unsigned char *p = lpFirst(lp);
aux = lpGetInteger(p); aux = lpGetInteger(p);
if (aux == 1) {
/* If this is the last element in the listpack, we can remove the whole
* node. */
lpFree(lp);
raxRemove(si->stream->rax,si->ri.key,si->ri.key_len,NULL);
} else {
/* In the base case we alter the counters of valid/deleted entries. */
lp = lpReplaceInteger(lp,&p,aux-1); lp = lpReplaceInteger(lp,&p,aux-1);
p = lpNext(lp,p); /* Seek deleted field. */ p = lpNext(lp,p); /* Seek deleted field. */
aux = lpGetInteger(p); aux = lpGetInteger(p);
lp = lpReplaceInteger(lp,&p,aux+1); lp = lpReplaceInteger(lp,&p,aux+1);
/* Update the listpack with the new pointer. */
if (si->lp != lp)
raxInsert(si->stream->rax,si->ri.key,si->ri.key_len,lp,NULL);
}
/* Update the number of entries counter. */ /* Update the number of entries counter. */
si->stream->length--; si->stream->length--;
...@@ -730,7 +751,7 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { ...@@ -730,7 +751,7 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) {
} }
/* Stop the stream iterator. The only cleanup we need is to free the rax /* Stop the stream iterator. The only cleanup we need is to free the rax
* itereator, since the stream iterator itself is supposed to be stack * iterator, since the stream iterator itself is supposed to be stack
* allocated. */ * allocated. */
void streamIteratorStop(streamIterator *si) { void streamIteratorStop(streamIterator *si) {
raxStop(&si->ri); raxStop(&si->ri);
...@@ -748,6 +769,7 @@ int streamDeleteItem(stream *s, streamID *id) { ...@@ -748,6 +769,7 @@ int streamDeleteItem(stream *s, streamID *id) {
streamIteratorRemoveEntry(&si,&myid); streamIteratorRemoveEntry(&si,&myid);
deleted = 1; deleted = 1;
} }
streamIteratorStop(&si);
return deleted; return deleted;
} }
...@@ -755,8 +777,8 @@ int streamDeleteItem(stream *s, streamID *id) { ...@@ -755,8 +777,8 @@ int streamDeleteItem(stream *s, streamID *id) {
* in the standard <ms>-<seq> format, using the simple string protocol * in the standard <ms>-<seq> format, using the simple string protocol
* of REPL. */ * of REPL. */
void addReplyStreamID(client *c, streamID *id) { void addReplyStreamID(client *c, streamID *id) {
sds replyid = sdscatfmt(sdsempty(),"+%U-%U\r\n",id->ms,id->seq); sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq);
addReplySds(c,replyid); addReplyBulkSds(c,replyid);
} }
/* Similar to the above function, but just creates an object, usually useful /* Similar to the above function, but just creates an object, usually useful
...@@ -769,18 +791,18 @@ robj *createObjectFromStreamID(streamID *id) { ...@@ -769,18 +791,18 @@ robj *createObjectFromStreamID(streamID *id) {
/* As a result of an explicit XCLAIM or XREADGROUP command, new entries /* As a result of an explicit XCLAIM or XREADGROUP command, new entries
* are created in the pending list of the stream and consumers. We need * are created in the pending list of the stream and consumers. We need
* to propagate this changes in the form of XCLAIM commands. */ * to propagate this changes in the form of XCLAIM commands. */
void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNACK *nack) { void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupname, robj *id, streamNACK *nack) {
/* We need to generate an XCLAIM that will work in a idempotent fashion: /* We need to generate an XCLAIM that will work in a idempotent fashion:
* *
* XCLAIM <key> <group> <consumer> 0 <id> TIME <milliseconds-unix-time> * XCLAIM <key> <group> <consumer> 0 <id> TIME <milliseconds-unix-time>
* RETRYCOUNT <count> FORCE JUSTID. * RETRYCOUNT <count> FORCE JUSTID LASTID <id>.
* *
* Note that JUSTID is useful in order to avoid that XCLAIM will do * Note that JUSTID is useful in order to avoid that XCLAIM will do
* useless work in the slave side, trying to fetch the stream item. */ * useless work in the slave side, trying to fetch the stream item. */
robj *argv[12]; robj *argv[14];
argv[0] = createStringObject("XCLAIM",6); argv[0] = createStringObject("XCLAIM",6);
argv[1] = key; argv[1] = key;
argv[2] = group; argv[2] = groupname;
argv[3] = createStringObject(nack->consumer->name,sdslen(nack->consumer->name)); argv[3] = createStringObject(nack->consumer->name,sdslen(nack->consumer->name));
argv[4] = createStringObjectFromLongLong(0); argv[4] = createStringObjectFromLongLong(0);
argv[5] = id; argv[5] = id;
...@@ -790,7 +812,9 @@ void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNA ...@@ -790,7 +812,9 @@ void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNA
argv[9] = createStringObjectFromLongLong(nack->delivery_count); argv[9] = createStringObjectFromLongLong(nack->delivery_count);
argv[10] = createStringObject("FORCE",5); argv[10] = createStringObject("FORCE",5);
argv[11] = createStringObject("JUSTID",6); argv[11] = createStringObject("JUSTID",6);
propagate(server.xclaimCommand,c->db->id,argv,12,PROPAGATE_AOF|PROPAGATE_REPL); argv[12] = createStringObject("LASTID",6);
argv[13] = createObjectFromStreamID(&group->last_id);
propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
decrRefCount(argv[0]); decrRefCount(argv[0]);
decrRefCount(argv[3]); decrRefCount(argv[3]);
decrRefCount(argv[4]); decrRefCount(argv[4]);
...@@ -800,13 +824,38 @@ void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNA ...@@ -800,13 +824,38 @@ void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNA
decrRefCount(argv[9]); decrRefCount(argv[9]);
decrRefCount(argv[10]); decrRefCount(argv[10]);
decrRefCount(argv[11]); decrRefCount(argv[11]);
decrRefCount(argv[12]);
decrRefCount(argv[13]);
}
/* We need this when we want to propoagate the new last-id of a consumer group
* that was consumed by XREADGROUP with the NOACK option: in that case we can't
* propagate the last ID just using the XCLAIM LASTID option, so we emit
*
* XGROUP SETID <key> <groupname> <id>
*/
void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupname) {
robj *argv[5];
argv[0] = createStringObject("XGROUP",6);
argv[1] = createStringObject("SETID",5);
argv[2] = key;
argv[3] = groupname;
argv[4] = createObjectFromStreamID(&group->last_id);
propagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL);
decrRefCount(argv[0]);
decrRefCount(argv[1]);
decrRefCount(argv[4]);
} }
/* Send the specified range to the client 'c'. The range the client will /* Send the stream items in the specified range to the client 'c'. The range
* receive is between start and end inclusive, if 'count' is non zero, no more * the client will receive is between start and end inclusive, if 'count' is
* than 'count' elemnets are sent. The 'end' pointer can be NULL to mean that * non zero, no more than 'count' elements are sent.
* we want all the elements from 'start' till the end of the stream. If 'rev' *
* is non zero, elements are produced in reversed order from end to start. * The 'end' pointer can be NULL to mean that we want all the elements from
* 'start' till the end of the stream. If 'rev' is non zero, elements are
* produced in reversed order from end to start.
*
* The function returns the number of entries emitted.
* *
* If group and consumer are not NULL, the function performs additional work: * If group and consumer are not NULL, the function performs additional work:
* 1. It updates the last delivered ID in the group in case we are * 1. It updates the last delivered ID in the group in case we are
...@@ -818,15 +867,15 @@ void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNA ...@@ -818,15 +867,15 @@ void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNA
* *
* The behavior may be modified passing non-zero flags: * The behavior may be modified passing non-zero flags:
* *
* STREAM_RWR_NOACK: Do not craete PEL entries, that is, the point "3" above * STREAM_RWR_NOACK: Do not create PEL entries, that is, the point "3" above
* is not performed. * is not performed.
* STREAM_RWR_RAWENTRIES: Do not emit array boundaries, but just the entries, * STREAM_RWR_RAWENTRIES: Do not emit array boundaries, but just the entries,
* and return the number of entries emitted as usually. * and return the number of entries emitted as usually.
* This is used when the function is just used in order * This is used when the function is just used in order
* to emit data and there is some higher level logic. * to emit data and there is some higher level logic.
* *
* The final argument 'spi' (stream propagatino info pointer) is a structure * The final argument 'spi' (stream propagation info pointer) is a structure
* filled with information needed to propagte the command execution to AOF * filled with information needed to propagate the command execution to AOF
* and slaves, in the case a consumer group was passed: we need to generate * and slaves, in the case a consumer group was passed: we need to generate
* XCLAIM commands to create the pending list into AOF/slaves in that case. * XCLAIM commands to create the pending list into AOF/slaves in that case.
* *
...@@ -834,7 +883,7 @@ void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNA ...@@ -834,7 +883,7 @@ void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNA
* given, but currently such a feature is never used by the code base that * given, but currently such a feature is never used by the code base that
* will always pass 'spi' and propagate when a group is passed. * will always pass 'spi' and propagate when a group is passed.
* *
* Note that this function is recursive in certian cases. When it's called * Note that this function is recursive in certain cases. When it's called
* with a non NULL group and consumer argument, it may call * with a non NULL group and consumer argument, it may call
* streamReplyWithRangeFromConsumerPEL() in order to get entries from the * streamReplyWithRangeFromConsumerPEL() in order to get entries from the
* consumer pending entries list. However such a function will then call * consumer pending entries list. However such a function will then call
...@@ -845,39 +894,41 @@ void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNA ...@@ -845,39 +894,41 @@ void streamPropagateXCLAIM(client *c, robj *key, robj *group, robj *id, streamNA
#define STREAM_RWR_NOACK (1<<0) /* Do not create entries in the PEL. */ #define STREAM_RWR_NOACK (1<<0) /* Do not create entries in the PEL. */
#define STREAM_RWR_RAWENTRIES (1<<1) /* Do not emit protocol for array #define STREAM_RWR_RAWENTRIES (1<<1) /* Do not emit protocol for array
boundaries, just the entries. */ boundaries, just the entries. */
#define STREAM_RWR_HISTORY (1<<2) /* Only serve consumer local PEL. */
size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end, size_t count, int rev, streamCG *group, streamConsumer *consumer, int flags, streamPropInfo *spi) { size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end, size_t count, int rev, streamCG *group, streamConsumer *consumer, int flags, streamPropInfo *spi) {
void *arraylen_ptr = NULL; void *arraylen_ptr = NULL;
size_t arraylen = 0; size_t arraylen = 0;
streamIterator si; streamIterator si;
int64_t numfields; int64_t numfields;
streamID id; streamID id;
int propagate_last_id = 0;
/* If a group was passed, we check if the request is about messages /* If the client is asking for some history, we serve it using a
* never delivered so far (normally this happens when ">" ID is passed). * different function, so that we return entries *solely* from its
* * own PEL. This ensures each consumer will always and only see
* If instead the client is asking for some history, we serve it * the history of messages delivered to it and not yet confirmed
* using a different function, so that we return entries *solely*
* from its own PEL. This ensures each consumer will always and only
* see the history of messages delivered to it and not yet confirmed
* as delivered. */ * as delivered. */
if (group && streamCompareID(start,&group->last_id) <= 0) { if (group && (flags & STREAM_RWR_HISTORY)) {
return streamReplyWithRangeFromConsumerPEL(c,s,start,end,count, return streamReplyWithRangeFromConsumerPEL(c,s,start,end,count,
consumer); consumer);
} }
if (!(flags & STREAM_RWR_RAWENTRIES)) if (!(flags & STREAM_RWR_RAWENTRIES))
arraylen_ptr = addDeferredMultiBulkLength(c); arraylen_ptr = addReplyDeferredLen(c);
streamIteratorStart(&si,s,start,end,rev); streamIteratorStart(&si,s,start,end,rev);
while(streamIteratorGetID(&si,&id,&numfields)) { while(streamIteratorGetID(&si,&id,&numfields)) {
/* Update the group last_id if needed. */ /* Update the group last_id if needed. */
if (group && streamCompareID(&id,&group->last_id) > 0) if (group && streamCompareID(&id,&group->last_id) > 0) {
group->last_id = id; group->last_id = id;
propagate_last_id = 1;
}
/* Emit a two elements array for each item. The first is /* Emit a two elements array for each item. The first is
* the ID, the second is an array of field-value pairs. */ * the ID, the second is an array of field-value pairs. */
addReplyMultiBulkLen(c,2); addReplyArrayLen(c,2);
addReplyStreamID(c,&id); addReplyStreamID(c,&id);
addReplyMultiBulkLen(c,numfields*2);
addReplyMapLen(c,numfields);
/* Emit the field-value pairs. */ /* Emit the field-value pairs. */
while(numfields--) { while(numfields--) {
...@@ -905,47 +956,52 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end ...@@ -905,47 +956,52 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
* will not require extra lookups. We'll fix the problem later * will not require extra lookups. We'll fix the problem later
* if we find that there is already a entry for this ID. */ * if we find that there is already a entry for this ID. */
streamNACK *nack = streamCreateNACK(consumer); streamNACK *nack = streamCreateNACK(consumer);
int retval = 0; int group_inserted =
retval += raxTryInsert(group->pel,buf,sizeof(buf),nack,NULL); raxTryInsert(group->pel,buf,sizeof(buf),nack,NULL);
retval += raxTryInsert(consumer->pel,buf,sizeof(buf),nack,NULL); int consumer_inserted =
raxTryInsert(consumer->pel,buf,sizeof(buf),nack,NULL);
/* Now we can check if the entry was already busy, and /* Now we can check if the entry was already busy, and
* in that case reassign the entry to the new consumer. */ * in that case reassign the entry to the new consumer,
if (retval == 0) { * or update it if the consumer is the same as before. */
if (group_inserted == 0) {
streamFreeNACK(nack); streamFreeNACK(nack);
nack = raxFind(group->pel,buf,sizeof(buf)); nack = raxFind(group->pel,buf,sizeof(buf));
serverAssert(nack != raxNotFound); serverAssert(nack != raxNotFound);
raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL); raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL);
/* Update the consumer and idle time. */ /* Update the consumer and NACK metadata. */
nack->consumer = consumer; nack->consumer = consumer;
nack->delivery_time = mstime(); nack->delivery_time = mstime();
nack->delivery_count++; nack->delivery_count = 1;
/* Add the entry in the new consumer local PEL. */ /* Add the entry in the new consumer local PEL. */
raxInsert(consumer->pel,buf,sizeof(buf),nack,NULL); raxInsert(consumer->pel,buf,sizeof(buf),nack,NULL);
} else if (retval == 1) { } else if (group_inserted == 1 && consumer_inserted == 0) {
serverPanic("NACK half-created. Should not be possible."); serverPanic("NACK half-created. Should not be possible.");
} }
/* Propagate as XCLAIM. */ /* Propagate as XCLAIM. */
if (spi) { if (spi) {
robj *idarg = createObjectFromStreamID(&id); robj *idarg = createObjectFromStreamID(&id);
streamPropagateXCLAIM(c,spi->keyname,spi->groupname,idarg,nack); streamPropagateXCLAIM(c,spi->keyname,group,spi->groupname,idarg,nack);
decrRefCount(idarg); decrRefCount(idarg);
} }
} else {
if (propagate_last_id)
streamPropagateGroupID(c,spi->keyname,group,spi->groupname);
} }
arraylen++; arraylen++;
if (count && count == arraylen) break; if (count && count == arraylen) break;
} }
streamIteratorStop(&si); streamIteratorStop(&si);
if (arraylen_ptr) setDeferredMultiBulkLength(c,arraylen_ptr,arraylen); if (arraylen_ptr) setDeferredArrayLen(c,arraylen_ptr,arraylen);
return arraylen; return arraylen;
} }
/* This is an helper function for streamReplyWithRange() when called with /* This is an helper function for streamReplyWithRange() when called with
* group and consumer arguments, but with a range that is referring to already * group and consumer arguments, but with a range that is referring to already
* delivered messages. In this case we just emit messages that are already * delivered messages. In this case we just emit messages that are already
* in the history of the conusmer, fetching the IDs from its PEL. * in the history of the consumer, fetching the IDs from its PEL.
* *
* Note that this function does not have a 'rev' argument because it's not * Note that this function does not have a 'rev' argument because it's not
* possible to iterate in reverse using a group. Basically this function * possible to iterate in reverse using a group. Basically this function
...@@ -963,25 +1019,25 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start ...@@ -963,25 +1019,25 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start
if (end) streamEncodeID(endkey,end); if (end) streamEncodeID(endkey,end);
size_t arraylen = 0; size_t arraylen = 0;
void *arraylen_ptr = addDeferredMultiBulkLength(c); void *arraylen_ptr = addReplyDeferredLen(c);
raxStart(&ri,consumer->pel); raxStart(&ri,consumer->pel);
raxSeek(&ri,">=",startkey,sizeof(startkey)); raxSeek(&ri,">=",startkey,sizeof(startkey));
while(raxNext(&ri) && (!count || arraylen < count)) { while(raxNext(&ri) && (!count || arraylen < count)) {
if (end && memcmp(ri.key,end,ri.key_len) > 0) break; if (end && memcmp(ri.key,end,ri.key_len) > 0) break;
streamID thisid; streamID thisid;
streamDecodeID(ri.key,&thisid); streamDecodeID(ri.key,&thisid);
if (streamReplyWithRange(c,s,&thisid,NULL,1,0,NULL,NULL, if (streamReplyWithRange(c,s,&thisid,&thisid,1,0,NULL,NULL,
STREAM_RWR_RAWENTRIES,NULL) == 0) STREAM_RWR_RAWENTRIES,NULL) == 0)
{ {
/* Note that we may have a not acknowledged entry in the PEL /* Note that we may have a not acknowledged entry in the PEL
* about a message that's no longer here because was removed * about a message that's no longer here because was removed
* by the user by other means. In that case we signal it emitting * by the user by other means. In that case we signal it emitting
* the ID but then a NULL entry for the fields. */ * the ID but then a NULL entry for the fields. */
addReplyMultiBulkLen(c,2); addReplyArrayLen(c,2);
streamID id; streamID id;
streamDecodeID(ri.key,&id); streamDecodeID(ri.key,&id);
addReplyStreamID(c,&id); addReplyStreamID(c,&id);
addReply(c,shared.nullmultibulk); addReplyNullArray(c);
} else { } else {
streamNACK *nack = ri.data; streamNACK *nack = ri.data;
nack->delivery_time = mstime(); nack->delivery_time = mstime();
...@@ -990,7 +1046,7 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start ...@@ -990,7 +1046,7 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start
arraylen++; arraylen++;
} }
raxStop(&ri); raxStop(&ri);
setDeferredMultiBulkLength(c,arraylen_ptr,arraylen); setDeferredArrayLen(c,arraylen_ptr,arraylen);
return arraylen; return arraylen;
} }
...@@ -1040,15 +1096,20 @@ int string2ull(const char *s, unsigned long long *value) { ...@@ -1040,15 +1096,20 @@ int string2ull(const char *s, unsigned long long *value) {
* to the client, otherwise C_OK is returned. The ID may be in incomplete * to the client, otherwise C_OK is returned. The ID may be in incomplete
* form, just stating the milliseconds time part of the stream. In such a case * form, just stating the milliseconds time part of the stream. In such a case
* the missing part is set according to the value of 'missing_seq' parameter. * the missing part is set according to the value of 'missing_seq' parameter.
*
* The IDs "-" and "+" specify respectively the minimum and maximum IDs * The IDs "-" and "+" specify respectively the minimum and maximum IDs
* that can be represented. * that can be represented. If 'strict' is set to 1, "-" and "+" will be
* treated as an invalid ID.
* *
* If 'c' is set to NULL, no reply is sent to the client. */ * If 'c' is set to NULL, no reply is sent to the client. */
int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq) { int streamGenericParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq, int strict) {
char buf[128]; char buf[128];
if (sdslen(o->ptr) > sizeof(buf)-1) goto invalid; if (sdslen(o->ptr) > sizeof(buf)-1) goto invalid;
memcpy(buf,o->ptr,sdslen(o->ptr)+1); memcpy(buf,o->ptr,sdslen(o->ptr)+1);
if (strict && (buf[0] == '-' || buf[0] == '+') && buf[1] == '\0')
goto invalid;
/* Handle the "-" and "+" special cases. */ /* Handle the "-" and "+" special cases. */
if (buf[0] == '-' && buf[1] == '\0') { if (buf[0] == '-' && buf[1] == '\0') {
id->ms = 0; id->ms = 0;
...@@ -1077,7 +1138,33 @@ invalid: ...@@ -1077,7 +1138,33 @@ invalid:
return C_ERR; return C_ERR;
} }
/* XADD key [MAXLEN <count>] <ID or *> [field value] [field value] ... */ /* Wrapper for streamGenericParseIDOrReply() with 'strict' argument set to
* 0, to be used when - and + are acceptable IDs. */
int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq) {
return streamGenericParseIDOrReply(c,o,id,missing_seq,0);
}
/* Wrapper for streamGenericParseIDOrReply() with 'strict' argument set to
* 1, to be used when we want to return an error if the special IDs + or -
* are provided. */
int streamParseStrictIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq) {
return streamGenericParseIDOrReply(c,o,id,missing_seq,1);
}
/* We propagate MAXLEN ~ <count> as MAXLEN = <resulting-len-of-stream>
* otherwise trimming is no longer determinsitic on replicas / AOF. */
void streamRewriteApproxMaxlen(client *c, stream *s, int maxlen_arg_idx) {
robj *maxlen_obj = createStringObjectFromLongLong(s->length);
robj *equal_obj = createStringObject("=",1);
rewriteClientCommandArgument(c,maxlen_arg_idx,maxlen_obj);
rewriteClientCommandArgument(c,maxlen_arg_idx-1,equal_obj);
decrRefCount(equal_obj);
decrRefCount(maxlen_obj);
}
/* XADD key [MAXLEN [~|=] <count>] <ID or *> [field value] [field value] ... */
void xaddCommand(client *c) { void xaddCommand(client *c) {
streamID id; streamID id;
int id_given = 0; /* Was an ID different than "*" specified? */ int id_given = 0; /* Was an ID different than "*" specified? */
...@@ -1097,11 +1184,14 @@ void xaddCommand(client *c) { ...@@ -1097,11 +1184,14 @@ void xaddCommand(client *c) {
* creation. */ * creation. */
break; break;
} else if (!strcasecmp(opt,"maxlen") && moreargs) { } else if (!strcasecmp(opt,"maxlen") && moreargs) {
approx_maxlen = 0;
char *next = c->argv[i+1]->ptr; char *next = c->argv[i+1]->ptr;
/* Check for the form MAXLEN ~ <count>. */ /* Check for the form MAXLEN ~ <count>. */
if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') { if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') {
approx_maxlen = 1; approx_maxlen = 1;
i++; i++;
} else if (moreargs >= 2 && next[0] == '=' && next[1] == '\0') {
i++;
} }
if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL) if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL)
!= C_OK) return; != C_OK) return;
...@@ -1114,7 +1204,7 @@ void xaddCommand(client *c) { ...@@ -1114,7 +1204,7 @@ void xaddCommand(client *c) {
maxlen_arg_idx = i; maxlen_arg_idx = i;
} else { } else {
/* If we are here is a syntax error or a valid ID. */ /* If we are here is a syntax error or a valid ID. */
if (streamParseIDOrReply(c,c->argv[i],&id,0) != C_OK) return; if (streamParseStrictIDOrReply(c,c->argv[i],&id,0) != C_OK) return;
id_given = 1; id_given = 1;
break; break;
} }
...@@ -1148,18 +1238,12 @@ void xaddCommand(client *c) { ...@@ -1148,18 +1238,12 @@ void xaddCommand(client *c) {
notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id);
server.dirty++; server.dirty++;
/* Remove older elements if MAXLEN was specified. */
if (maxlen >= 0) { if (maxlen >= 0) {
if (!streamTrimByLength(s,maxlen,approx_maxlen)) { /* Notify xtrim event if needed. */
/* If no trimming was performed, for instance because approximated if (streamTrimByLength(s,maxlen,approx_maxlen)) {
* trimming length was specified, rewrite the MAXLEN argument
* as zero, so that the command is propagated without trimming. */
robj *zeroobj = createStringObjectFromLongLong(0);
rewriteClientCommandArgument(c,maxlen_arg_idx,zeroobj);
decrRefCount(zeroobj);
} else {
notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id);
} }
if (approx_maxlen) streamRewriteApproxMaxlen(c,s,maxlen_arg_idx);
} }
/* Let's rewrite the ID argument with the one actually generated for /* Let's rewrite the ID argument with the one actually generated for
...@@ -1179,7 +1263,7 @@ void xrangeGenericCommand(client *c, int rev) { ...@@ -1179,7 +1263,7 @@ void xrangeGenericCommand(client *c, int rev) {
robj *o; robj *o;
stream *s; stream *s;
streamID startid, endid; streamID startid, endid;
long long count = 0; long long count = -1;
robj *startarg = rev ? c->argv[3] : c->argv[2]; robj *startarg = rev ? c->argv[3] : c->argv[2];
robj *endarg = rev ? c->argv[2] : c->argv[3]; robj *endarg = rev ? c->argv[2] : c->argv[3];
...@@ -1203,10 +1287,17 @@ void xrangeGenericCommand(client *c, int rev) { ...@@ -1203,10 +1287,17 @@ void xrangeGenericCommand(client *c, int rev) {
} }
/* Return the specified range to the user. */ /* Return the specified range to the user. */
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) == NULL if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptyarray)) == NULL ||
|| checkType(c,o,OBJ_STREAM)) return; checkType(c,o,OBJ_STREAM)) return;
s = o->ptr; s = o->ptr;
if (count == 0) {
addReplyNullArray(c);
} else {
if (count == -1) count = 0;
streamReplyWithRange(c,s,&startid,&endid,count,rev,NULL,NULL,0,NULL); streamReplyWithRange(c,s,&startid,&endid,count,rev,NULL,NULL,0,NULL);
}
} }
/* XRANGE key start end [COUNT <n>] */ /* XRANGE key start end [COUNT <n>] */
...@@ -1340,6 +1431,14 @@ void xreadCommand(client *c) { ...@@ -1340,6 +1431,14 @@ void xreadCommand(client *c) {
} }
if (strcmp(c->argv[i]->ptr,"$") == 0) { if (strcmp(c->argv[i]->ptr,"$") == 0) {
if (xreadgroup) {
addReplyError(c,"The $ ID is meaningless in the context of "
"XREADGROUP: you want to read the history of "
"this consumer by specifying a proper ID, or "
"use the > ID to get new messages. The $ ID would "
"just return an empty result set.");
goto cleanup;
}
if (o) { if (o) {
stream *s = o->ptr; stream *s = o->ptr;
ids[id_idx] = s->last_id; ids[id_idx] = s->last_id;
...@@ -1349,16 +1448,20 @@ void xreadCommand(client *c) { ...@@ -1349,16 +1448,20 @@ void xreadCommand(client *c) {
} }
continue; continue;
} else if (strcmp(c->argv[i]->ptr,">") == 0) { } else if (strcmp(c->argv[i]->ptr,">") == 0) {
if (!xreadgroup || groupname == NULL) { if (!xreadgroup) {
addReplyError(c,"The > ID can be specified only when calling " addReplyError(c,"The > ID can be specified only when calling "
"XREADGROUP using the GROUP <group> " "XREADGROUP using the GROUP <group> "
"<consumer> option."); "<consumer> option.");
goto cleanup; goto cleanup;
} }
ids[id_idx] = group->last_id; /* We use just the maximum ID to signal this is a ">" ID, anyway
* the code handling the blocking clients will have to update the
* ID later in order to match the changing consumer group last ID. */
ids[id_idx].ms = UINT64_MAX;
ids[id_idx].seq = UINT64_MAX;
continue; continue;
} }
if (streamParseIDOrReply(c,c->argv[i],ids+id_idx,0) != C_OK) if (streamParseStrictIDOrReply(c,c->argv[i],ids+id_idx,0) != C_OK)
goto cleanup; goto cleanup;
} }
...@@ -1370,11 +1473,41 @@ void xreadCommand(client *c) { ...@@ -1370,11 +1473,41 @@ void xreadCommand(client *c) {
if (o == NULL) continue; if (o == NULL) continue;
stream *s = o->ptr; stream *s = o->ptr;
streamID *gt = ids+i; /* ID must be greater than this. */ streamID *gt = ids+i; /* ID must be greater than this. */
if (s->last_id.ms > gt->ms || int serve_synchronously = 0;
(s->last_id.ms == gt->ms && s->last_id.seq > gt->seq)) int serve_history = 0; /* True for XREADGROUP with ID != ">". */
/* Check if there are the conditions to serve the client
* synchronously. */
if (groups) {
/* If the consumer is blocked on a group, we always serve it
* synchronously (serving its local history) if the ID specified
* was not the special ">" ID. */
if (gt->ms != UINT64_MAX ||
gt->seq != UINT64_MAX)
{ {
serve_synchronously = 1;
serve_history = 1;
} else {
/* We also want to serve a consumer in a consumer group
* synchronously in case the group top item delivered is smaller
* than what the stream has inside. */
streamID *last = &groups[i]->last_id;
if (s->length && (streamCompareID(&s->last_id, last) > 0)) {
serve_synchronously = 1;
*gt = *last;
}
}
} else {
/* For consumers without a group, we serve synchronously if we can
* actually provide at least one item from the stream. */
if (s->length && (streamCompareID(&s->last_id, gt) > 0)) {
serve_synchronously = 1;
}
}
if (serve_synchronously) {
arraylen++; arraylen++;
if (arraylen == 1) arraylen_ptr = addDeferredMultiBulkLength(c); if (arraylen == 1) arraylen_ptr = addReplyDeferredLen(c);
/* streamReplyWithRange() handles the 'start' ID as inclusive, /* streamReplyWithRange() handles the 'start' ID as inclusive,
* so start from the next ID, since we want only messages with * so start from the next ID, since we want only messages with
* IDs greater than start. */ * IDs greater than start. */
...@@ -1383,22 +1516,28 @@ void xreadCommand(client *c) { ...@@ -1383,22 +1516,28 @@ void xreadCommand(client *c) {
/* Emit the two elements sub-array consisting of the name /* Emit the two elements sub-array consisting of the name
* of the stream and the data we extracted from it. */ * of the stream and the data we extracted from it. */
addReplyMultiBulkLen(c,2); if (c->resp == 2) addReplyArrayLen(c,2);
addReplyBulk(c,c->argv[streams_arg+i]); addReplyBulk(c,c->argv[streams_arg+i]);
streamConsumer *consumer = NULL; streamConsumer *consumer = NULL;
if (groups) consumer = streamLookupConsumer(groups[i], if (groups) consumer = streamLookupConsumer(groups[i],
consumername->ptr,1); consumername->ptr,1);
streamPropInfo spi = {c->argv[i+streams_arg],groupname}; streamPropInfo spi = {c->argv[i+streams_arg],groupname};
int flags = 0;
if (noack) flags |= STREAM_RWR_NOACK;
if (serve_history) flags |= STREAM_RWR_HISTORY;
streamReplyWithRange(c,s,&start,NULL,count,0, streamReplyWithRange(c,s,&start,NULL,count,0,
groups ? groups[i] : NULL, groups ? groups[i] : NULL,
consumer, noack, &spi); consumer, flags, &spi);
if (groups) server.dirty++; if (groups) server.dirty++;
} }
} }
/* We replied synchronously! Set the top array len and return to caller. */ /* We replied synchronously! Set the top array len and return to caller. */
if (arraylen) { if (arraylen) {
setDeferredMultiBulkLength(c,arraylen_ptr,arraylen); if (c->resp == 2)
setDeferredArrayLen(c,arraylen_ptr,arraylen);
else
setDeferredMapLen(c,arraylen_ptr,arraylen);
goto cleanup; goto cleanup;
} }
...@@ -1407,7 +1546,7 @@ void xreadCommand(client *c) { ...@@ -1407,7 +1546,7 @@ void xreadCommand(client *c) {
/* If we are inside a MULTI/EXEC and the list is empty the only thing /* If we are inside a MULTI/EXEC and the list is empty the only thing
* we can do is treating it as a timeout (even with timeout 0). */ * we can do is treating it as a timeout (even with timeout 0). */
if (c->flags & CLIENT_MULTI) { if (c->flags & CLIENT_MULTI) {
addReply(c,shared.nullmultibulk); addReplyNullArray(c);
goto cleanup; goto cleanup;
} }
blockForKeys(c, BLOCKED_STREAM, c->argv+streams_arg, streams_count, blockForKeys(c, BLOCKED_STREAM, c->argv+streams_arg, streams_count,
...@@ -1426,6 +1565,7 @@ void xreadCommand(client *c) { ...@@ -1426,6 +1565,7 @@ void xreadCommand(client *c) {
incrRefCount(consumername); incrRefCount(consumername);
c->bpop.xread_group = groupname; c->bpop.xread_group = groupname;
c->bpop.xread_consumer = consumername; c->bpop.xread_consumer = consumername;
c->bpop.xread_group_noack = noack;
} else { } else {
c->bpop.xread_group = NULL; c->bpop.xread_group = NULL;
c->bpop.xread_consumer = NULL; c->bpop.xread_consumer = NULL;
...@@ -1435,7 +1575,7 @@ void xreadCommand(client *c) { ...@@ -1435,7 +1575,7 @@ void xreadCommand(client *c) {
/* No BLOCK option, nor any stream we can serve. Reply as with a /* No BLOCK option, nor any stream we can serve. Reply as with a
* timeout happened. */ * timeout happened. */
addReply(c,shared.nullmultibulk); addReplyNullArray(c);
/* Continue to cleanup... */ /* Continue to cleanup... */
cleanup: /* Cleanup. */ cleanup: /* Cleanup. */
...@@ -1562,16 +1702,17 @@ uint64_t streamDelConsumer(streamCG *cg, sds name) { ...@@ -1562,16 +1702,17 @@ uint64_t streamDelConsumer(streamCG *cg, sds name) {
* Consumer groups commands * Consumer groups commands
* ----------------------------------------------------------------------- */ * ----------------------------------------------------------------------- */
/* XGROUP CREATE <key> <groupname> <id or $> /* XGROUP CREATE <key> <groupname> <id or $> [MKSTREAM]
* XGROUP SETID <key> <id or $> * XGROUP SETID <key> <groupname> <id or $>
* XGROUP DESTROY <key> <groupname> * XGROUP DESTROY <key> <groupname>
* XGROUP DELCONSUMER <key> <groupname> <consumername> */ * XGROUP DELCONSUMER <key> <groupname> <consumername> */
void xgroupCommand(client *c) { void xgroupCommand(client *c) {
const char *help[] = { const char *help[] = {
"CREATE <key> <groupname> <id or $> -- Create a new consumer group.", "CREATE <key> <groupname> <id or $> [opt] -- Create a new consumer group.",
" option MKSTREAM: create the empty stream if it does not exist.",
"SETID <key> <groupname> <id or $> -- Set the current group ID.", "SETID <key> <groupname> <id or $> -- Set the current group ID.",
"DESTROY <key> <groupname> -- Remove the specified group.", "DESTROY <key> <groupname> -- Remove the specified group.",
"DELCONSUMER <key> <groupname> <consumer> -- Remove the specified conusmer.", "DELCONSUMER <key> <groupname> <consumer> -- Remove the specified consumer.",
"HELP -- Prints this help.", "HELP -- Prints this help.",
NULL NULL
}; };
...@@ -1579,13 +1720,40 @@ NULL ...@@ -1579,13 +1720,40 @@ NULL
sds grpname = NULL; sds grpname = NULL;
streamCG *cg = NULL; streamCG *cg = NULL;
char *opt = c->argv[1]->ptr; /* Subcommand name. */ char *opt = c->argv[1]->ptr; /* Subcommand name. */
int mkstream = 0;
robj *o;
/* Lookup the key now, this is common for all the subcommands but HELP. */ /* CREATE has an MKSTREAM option that creates the stream if it
* does not exist. */
if (c->argc == 6 && !strcasecmp(opt,"CREATE")) {
if (strcasecmp(c->argv[5]->ptr,"MKSTREAM")) {
addReplySubcommandSyntaxError(c);
return;
}
mkstream = 1;
grpname = c->argv[3]->ptr;
}
/* Everything but the "HELP" option requires a key and group name. */
if (c->argc >= 4) { if (c->argc >= 4) {
robj *o = lookupKeyWriteOrReply(c,c->argv[2],shared.nokeyerr); o = lookupKeyWrite(c->db,c->argv[2]);
if (o == NULL || checkType(c,o,OBJ_STREAM)) return; if (o) {
if (checkType(c,o,OBJ_STREAM)) return;
s = o->ptr; s = o->ptr;
}
grpname = c->argv[3]->ptr; grpname = c->argv[3]->ptr;
}
/* Check for missing key/group. */
if (c->argc >= 4 && !mkstream) {
/* At this point key must exist, or there is an error. */
if (s == NULL) {
addReplyError(c,
"The XGROUP subcommand requires the key to exist. "
"Note that for CREATE you may want to use the MKSTREAM "
"option to create an empty stream automatically.");
return;
}
/* Certain subcommands require the group to exist. */ /* Certain subcommands require the group to exist. */
if ((cg = streamLookupCG(s,grpname)) == NULL && if ((cg = streamLookupCG(s,grpname)) == NULL &&
...@@ -1600,13 +1768,27 @@ NULL ...@@ -1600,13 +1768,27 @@ NULL
} }
/* Dispatch the different subcommands. */ /* Dispatch the different subcommands. */
if (!strcasecmp(opt,"CREATE") && c->argc == 5) { if (!strcasecmp(opt,"CREATE") && (c->argc == 5 || c->argc == 6)) {
streamID id; streamID id;
if (!strcmp(c->argv[4]->ptr,"$")) { if (!strcmp(c->argv[4]->ptr,"$")) {
if (s) {
id = s->last_id; id = s->last_id;
} else if (streamParseIDOrReply(c,c->argv[4],&id,0) != C_OK) { } else {
id.ms = 0;
id.seq = 0;
}
} else if (streamParseStrictIDOrReply(c,c->argv[4],&id,0) != C_OK) {
return; return;
} }
/* Handle the MKSTREAM option now that the command can no longer fail. */
if (s == NULL) {
serverAssert(mkstream);
o = createStreamObject();
dbAdd(c->db,c->argv[2],o);
s = o->ptr;
}
streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id); streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id);
if (cg) { if (cg) {
addReply(c,shared.ok); addReply(c,shared.ok);
...@@ -1650,8 +1832,42 @@ NULL ...@@ -1650,8 +1832,42 @@ NULL
} else if (!strcasecmp(opt,"HELP")) { } else if (!strcasecmp(opt,"HELP")) {
addReplyHelp(c, help); addReplyHelp(c, help);
} else { } else {
addReply(c,shared.syntaxerr); addReplySubcommandSyntaxError(c);
}
}
/* XSETID <stream> <groupname> <id>
*
* Set the internal "last ID" of a stream. */
void xsetidCommand(client *c) {
robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr);
if (o == NULL || checkType(c,o,OBJ_STREAM)) return;
stream *s = o->ptr;
streamID id;
if (streamParseStrictIDOrReply(c,c->argv[2],&id,0) != C_OK) return;
/* If the stream has at least one item, we want to check that the user
* is setting a last ID that is equal or greater than the current top
* item, otherwise the fundamental ID monotonicity assumption is violated. */
if (s->length > 0) {
streamID maxid;
streamIterator si;
streamIteratorStart(&si,s,NULL,NULL,1);
int64_t numfields;
streamIteratorGetID(&si,&maxid,&numfields);
streamIteratorStop(&si);
if (streamCompareID(&id,&maxid) < 0) {
addReplyError(c,"The ID specified in XSETID is smaller than the "
"target stream top item");
return;
} }
}
s->last_id = id;
addReply(c,shared.ok);
server.dirty++;
notifyKeyspaceEvent(NOTIFY_STREAM,"xsetid",c->argv[1],c->db->id);
} }
/* XACK <key> <group> <id> <id> ... <id> /* XACK <key> <group> <id> <id> ... <id>
...@@ -1681,7 +1897,7 @@ void xackCommand(client *c) { ...@@ -1681,7 +1897,7 @@ void xackCommand(client *c) {
for (int j = 3; j < c->argc; j++) { for (int j = 3; j < c->argc; j++) {
streamID id; streamID id;
unsigned char buf[sizeof(streamID)]; unsigned char buf[sizeof(streamID)];
if (streamParseIDOrReply(c,c->argv[j],&id,0) != C_OK) return; if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return;
streamEncodeID(buf,&id); streamEncodeID(buf,&id);
/* Lookup the ID in the group PEL: it will have a reference to the /* Lookup the ID in the group PEL: it will have a reference to the
...@@ -1699,7 +1915,7 @@ void xackCommand(client *c) { ...@@ -1699,7 +1915,7 @@ void xackCommand(client *c) {
addReplyLongLong(c,acknowledged); addReplyLongLong(c,acknowledged);
} }
/* XPENDING <key> <group> [<start> <stop> <count>] [<consumer>] /* XPENDING <key> <group> [<start> <stop> <count> [<consumer>]]
* *
* If start and stop are omitted, the command just outputs information about * If start and stop are omitted, the command just outputs information about
* the amount of pending messages for the key/group pair, together with * the amount of pending messages for the key/group pair, together with
...@@ -1728,6 +1944,7 @@ void xpendingCommand(client *c) { ...@@ -1728,6 +1944,7 @@ void xpendingCommand(client *c) {
if (c->argc >= 6) { if (c->argc >= 6) {
if (getLongLongFromObjectOrReply(c,c->argv[5],&count,NULL) == C_ERR) if (getLongLongFromObjectOrReply(c,c->argv[5],&count,NULL) == C_ERR)
return; return;
if (count < 0) count = 0;
if (streamParseIDOrReply(c,c->argv[3],&startid,0) == C_ERR) if (streamParseIDOrReply(c,c->argv[3],&startid,0) == C_ERR)
return; return;
if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX) == C_ERR) if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX) == C_ERR)
...@@ -1750,14 +1967,14 @@ void xpendingCommand(client *c) { ...@@ -1750,14 +1967,14 @@ void xpendingCommand(client *c) {
/* XPENDING <key> <group> variant. */ /* XPENDING <key> <group> variant. */
if (justinfo) { if (justinfo) {
addReplyMultiBulkLen(c,4); addReplyArrayLen(c,4);
/* Total number of messages in the PEL. */ /* Total number of messages in the PEL. */
addReplyLongLong(c,raxSize(group->pel)); addReplyLongLong(c,raxSize(group->pel));
/* First and last IDs. */ /* First and last IDs. */
if (raxSize(group->pel) == 0) { if (raxSize(group->pel) == 0) {
addReply(c,shared.nullbulk); /* Start. */ addReplyNull(c); /* Start. */
addReply(c,shared.nullbulk); /* End. */ addReplyNull(c); /* End. */
addReply(c,shared.nullmultibulk); /* Clients. */ addReplyNullArray(c); /* Clients. */
} else { } else {
/* Start. */ /* Start. */
raxIterator ri; raxIterator ri;
...@@ -1777,17 +1994,17 @@ void xpendingCommand(client *c) { ...@@ -1777,17 +1994,17 @@ void xpendingCommand(client *c) {
/* Consumers with pending messages. */ /* Consumers with pending messages. */
raxStart(&ri,group->consumers); raxStart(&ri,group->consumers);
raxSeek(&ri,"^",NULL,0); raxSeek(&ri,"^",NULL,0);
void *arraylen_ptr = addDeferredMultiBulkLength(c); void *arraylen_ptr = addReplyDeferredLen(c);
size_t arraylen = 0; size_t arraylen = 0;
while(raxNext(&ri)) { while(raxNext(&ri)) {
streamConsumer *consumer = ri.data; streamConsumer *consumer = ri.data;
if (raxSize(consumer->pel) == 0) continue; if (raxSize(consumer->pel) == 0) continue;
addReplyMultiBulkLen(c,2); addReplyArrayLen(c,2);
addReplyBulkCBuffer(c,ri.key,ri.key_len); addReplyBulkCBuffer(c,ri.key,ri.key_len);
addReplyBulkLongLong(c,raxSize(consumer->pel)); addReplyBulkLongLong(c,raxSize(consumer->pel));
arraylen++; arraylen++;
} }
setDeferredMultiBulkLength(c,arraylen_ptr,arraylen); setDeferredArrayLen(c,arraylen_ptr,arraylen);
raxStop(&ri); raxStop(&ri);
} }
} }
...@@ -1800,7 +2017,7 @@ void xpendingCommand(client *c) { ...@@ -1800,7 +2017,7 @@ void xpendingCommand(client *c) {
/* If a consumer name was mentioned but it does not exist, we can /* If a consumer name was mentioned but it does not exist, we can
* just return an empty array. */ * just return an empty array. */
if (consumername && consumer == NULL) { if (consumername && consumer == NULL) {
addReplyMultiBulkLen(c,0); addReplyArrayLen(c,0);
return; return;
} }
...@@ -1814,7 +2031,7 @@ void xpendingCommand(client *c) { ...@@ -1814,7 +2031,7 @@ void xpendingCommand(client *c) {
streamEncodeID(endkey,&endid); streamEncodeID(endkey,&endid);
raxStart(&ri,pel); raxStart(&ri,pel);
raxSeek(&ri,">=",startkey,sizeof(startkey)); raxSeek(&ri,">=",startkey,sizeof(startkey));
void *arraylen_ptr = addDeferredMultiBulkLength(c); void *arraylen_ptr = addReplyDeferredLen(c);
size_t arraylen = 0; size_t arraylen = 0;
while(count && raxNext(&ri) && memcmp(ri.key,endkey,ri.key_len) <= 0) { while(count && raxNext(&ri) && memcmp(ri.key,endkey,ri.key_len) <= 0) {
...@@ -1822,7 +2039,7 @@ void xpendingCommand(client *c) { ...@@ -1822,7 +2039,7 @@ void xpendingCommand(client *c) {
arraylen++; arraylen++;
count--; count--;
addReplyMultiBulkLen(c,4); addReplyArrayLen(c,4);
/* Entry ID. */ /* Entry ID. */
streamID id; streamID id;
...@@ -1842,7 +2059,7 @@ void xpendingCommand(client *c) { ...@@ -1842,7 +2059,7 @@ void xpendingCommand(client *c) {
addReplyLongLong(c,nack->delivery_count); addReplyLongLong(c,nack->delivery_count);
} }
raxStop(&ri); raxStop(&ri);
setDeferredMultiBulkLength(c,arraylen_ptr,arraylen); setDeferredArrayLen(c,arraylen_ptr,arraylen);
} }
} }
...@@ -1901,6 +2118,14 @@ void xpendingCommand(client *c) { ...@@ -1901,6 +2118,14 @@ void xpendingCommand(client *c) {
* Return just an array of IDs of messages successfully claimed, * Return just an array of IDs of messages successfully claimed,
* without returning the actual message. * without returning the actual message.
* *
* 6. LASTID <id>:
* Update the consumer group last ID with the specified ID if the
* current last ID is smaller than the provided one.
* This is used for replication / AOF, so that when we read from a
* consumer group, the XCLAIM that gets propagated to give ownership
* to the consumer, is also used in order to update the group current
* ID.
*
* The command returns an array of messages that the user * The command returns an array of messages that the user
* successfully claimed, so that the caller is able to understand * successfully claimed, so that the caller is able to understand
* what messages it is now in charge of. */ * what messages it is now in charge of. */
...@@ -1937,15 +2162,17 @@ void xclaimCommand(client *c) { ...@@ -1937,15 +2162,17 @@ void xclaimCommand(client *c) {
* the client successfully claimed some message, so it should be * the client successfully claimed some message, so it should be
* executed in a "all or nothing" fashion. */ * executed in a "all or nothing" fashion. */
int j; int j;
for (j = 4; j < c->argc; j++) { for (j = 5; j < c->argc; j++) {
streamID id; streamID id;
if (streamParseIDOrReply(NULL,c->argv[j],&id,0) != C_OK) break; if (streamParseStrictIDOrReply(NULL,c->argv[j],&id,0) != C_OK) break;
} }
int last_id_arg = j-1; /* Next time we iterate the IDs we now the range. */ int last_id_arg = j-1; /* Next time we iterate the IDs we now the range. */
/* If we stopped because some IDs cannot be parsed, perhaps they /* If we stopped because some IDs cannot be parsed, perhaps they
* are trailing options. */ * are trailing options. */
time_t now = mstime(); mstime_t now = mstime();
streamID last_id = {0,0};
int propagate_last_id = 0;
for (; j < c->argc; j++) { for (; j < c->argc; j++) {
int moreargs = (c->argc-1) - j; /* Number of additional arguments. */ int moreargs = (c->argc-1) - j; /* Number of additional arguments. */
char *opt = c->argv[j]->ptr; char *opt = c->argv[j]->ptr;
...@@ -1962,27 +2189,35 @@ void xclaimCommand(client *c) { ...@@ -1962,27 +2189,35 @@ void xclaimCommand(client *c) {
} else if (!strcasecmp(opt,"TIME") && moreargs) { } else if (!strcasecmp(opt,"TIME") && moreargs) {
j++; j++;
if (getLongLongFromObjectOrReply(c,c->argv[j],&deliverytime, if (getLongLongFromObjectOrReply(c,c->argv[j],&deliverytime,
"Invalid IDLE option argument for XCLAIM") "Invalid TIME option argument for XCLAIM")
!= C_OK) return; != C_OK) return;
} else if (!strcasecmp(opt,"RETRYCOUNT") && moreargs) { } else if (!strcasecmp(opt,"RETRYCOUNT") && moreargs) {
j++; j++;
if (getLongLongFromObjectOrReply(c,c->argv[j],&retrycount, if (getLongLongFromObjectOrReply(c,c->argv[j],&retrycount,
"Invalid IDLE option argument for XCLAIM") "Invalid RETRYCOUNT option argument for XCLAIM")
!= C_OK) return; != C_OK) return;
} else if (!strcasecmp(opt,"LASTID") && moreargs) {
j++;
if (streamParseStrictIDOrReply(c,c->argv[j],&last_id,0) != C_OK) return;
} else { } else {
addReplyErrorFormat(c,"Unrecognized XCLAIM option '%s'",opt); addReplyErrorFormat(c,"Unrecognized XCLAIM option '%s'",opt);
return; return;
} }
} }
if (streamCompareID(&last_id,&group->last_id) > 0) {
group->last_id = last_id;
propagate_last_id = 1;
}
if (deliverytime != -1) { if (deliverytime != -1) {
/* If a delivery time was passed, either with IDLE or TIME, we /* If a delivery time was passed, either with IDLE or TIME, we
* do some sanity check on it, and set the deliverytime to now * do some sanity check on it, and set the deliverytime to now
* (which is a sane choice usually) if the value is bogus. * (which is a sane choice usually) if the value is bogus.
* To raise an error here is not wise because clients may compute * To raise an error here is not wise because clients may compute
* the idle time doing some math startin from their local time, * the idle time doing some math starting from their local time,
* and this is not a good excuse to fail in case, for instance, * and this is not a good excuse to fail in case, for instance,
* the computed time is a bit in the future from our POV. */ * the computer time is a bit in the future from our POV. */
if (deliverytime < 0 || deliverytime > now) deliverytime = now; if (deliverytime < 0 || deliverytime > now) deliverytime = now;
} else { } else {
/* If no IDLE/TIME option was passed, we want the last delivery /* If no IDLE/TIME option was passed, we want the last delivery
...@@ -1993,12 +2228,13 @@ void xclaimCommand(client *c) { ...@@ -1993,12 +2228,13 @@ void xclaimCommand(client *c) {
/* Do the actual claiming. */ /* Do the actual claiming. */
streamConsumer *consumer = streamLookupConsumer(group,c->argv[3]->ptr,1); streamConsumer *consumer = streamLookupConsumer(group,c->argv[3]->ptr,1);
void *arraylenptr = addDeferredMultiBulkLength(c); void *arraylenptr = addReplyDeferredLen(c);
size_t arraylen = 0; size_t arraylen = 0;
for (int j = 5; j <= last_id_arg; j++) { for (int j = 5; j <= last_id_arg; j++) {
streamID id; streamID id;
unsigned char buf[sizeof(streamID)]; unsigned char buf[sizeof(streamID)];
if (streamParseIDOrReply(c,c->argv[j],&id,0) != C_OK) return; if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK)
serverPanic("StreamID invalid after check. Should not be possible.");
streamEncodeID(buf,&id); streamEncodeID(buf,&id);
/* Lookup the ID in the group PEL. */ /* Lookup the ID in the group PEL. */
...@@ -2028,8 +2264,12 @@ void xclaimCommand(client *c) { ...@@ -2028,8 +2264,12 @@ void xclaimCommand(client *c) {
if (nack != raxNotFound) { if (nack != raxNotFound) {
/* We need to check if the minimum idle time requested /* We need to check if the minimum idle time requested
* by the caller is satisfied by this entry. */ * by the caller is satisfied by this entry.
if (minidle) { *
* Note that the nack could be created by FORCE, in this
* case there was no pre-existing entry and minidle should
* be ignored, but in that case nick->consumer is NULL. */
if (nack->consumer && minidle) {
mstime_t this_idle = now - nack->delivery_time; mstime_t this_idle = now - nack->delivery_time;
if (this_idle < minidle) continue; if (this_idle < minidle) continue;
} }
...@@ -2041,25 +2281,36 @@ void xclaimCommand(client *c) { ...@@ -2041,25 +2281,36 @@ void xclaimCommand(client *c) {
/* Update the consumer and idle time. */ /* Update the consumer and idle time. */
nack->consumer = consumer; nack->consumer = consumer;
nack->delivery_time = deliverytime; nack->delivery_time = deliverytime;
/* Set the delivery attempts counter if given. */ /* Set the delivery attempts counter if given, otherwise
if (retrycount >= 0) nack->delivery_count = retrycount; * autoincrement unless JUSTID option provided */
if (retrycount >= 0) {
nack->delivery_count = retrycount;
} else if (!justid) {
nack->delivery_count++;
}
/* Add the entry in the new consumer local PEL. */ /* Add the entry in the new consumer local PEL. */
raxInsert(consumer->pel,buf,sizeof(buf),nack,NULL); raxInsert(consumer->pel,buf,sizeof(buf),nack,NULL);
/* Send the reply for this entry. */ /* Send the reply for this entry. */
if (justid) { if (justid) {
addReplyStreamID(c,&id); addReplyStreamID(c,&id);
} else { } else {
streamReplyWithRange(c,o->ptr,&id,NULL,1,0,NULL,NULL, size_t emitted = streamReplyWithRange(c,o->ptr,&id,&id,1,0,
STREAM_RWR_RAWENTRIES,NULL); NULL,NULL,STREAM_RWR_RAWENTRIES,NULL);
if (!emitted) addReplyNull(c);
} }
arraylen++; arraylen++;
/* Propagate this change. */ /* Propagate this change. */
streamPropagateXCLAIM(c,c->argv[1],c->argv[3],c->argv[j],nack); streamPropagateXCLAIM(c,c->argv[1],group,c->argv[2],c->argv[j],nack);
propagate_last_id = 0; /* Will be propagated by XCLAIM itself. */
server.dirty++; server.dirty++;
} }
} }
setDeferredMultiBulkLength(c,arraylenptr,arraylen); if (propagate_last_id) {
streamPropagateGroupID(c,c->argv[1],group,c->argv[2]);
server.dirty++;
}
setDeferredArrayLen(c,arraylenptr,arraylen);
preventCommandPropagation(c); preventCommandPropagation(c);
} }
...@@ -2067,7 +2318,7 @@ void xclaimCommand(client *c) { ...@@ -2067,7 +2318,7 @@ void xclaimCommand(client *c) {
/* XDEL <key> [<ID1> <ID2> ... <IDN>] /* XDEL <key> [<ID1> <ID2> ... <IDN>]
* *
* Removes the specified entries from the stream. Returns the number * Removes the specified entries from the stream. Returns the number
* of items actaully deleted, that may be different from the number * of items actually deleted, that may be different from the number
* of IDs passed in case certain IDs do not exist. */ * of IDs passed in case certain IDs do not exist. */
void xdelCommand(client *c) { void xdelCommand(client *c) {
robj *o; robj *o;
...@@ -2078,21 +2329,25 @@ void xdelCommand(client *c) { ...@@ -2078,21 +2329,25 @@ void xdelCommand(client *c) {
/* We need to sanity check the IDs passed to start. Even if not /* We need to sanity check the IDs passed to start. Even if not
* a big issue, it is not great that the command is only partially * a big issue, it is not great that the command is only partially
* executed becuase at some point an invalid ID is parsed. */ * executed because at some point an invalid ID is parsed. */
streamID id; streamID id;
for (int j = 2; j < c->argc; j++) { for (int j = 2; j < c->argc; j++) {
if (streamParseIDOrReply(c,c->argv[j],&id,0) != C_OK) return; if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return;
} }
/* Actaully apply the command. */ /* Actually apply the command. */
int deleted = 0; int deleted = 0;
for (int j = 2; j < c->argc; j++) { for (int j = 2; j < c->argc; j++) {
streamParseIDOrReply(c,c->argv[j],&id,0); /* Retval already checked. */ streamParseStrictIDOrReply(c,c->argv[j],&id,0); /* Retval already checked. */
deleted += streamDeleteItem(s,&id); deleted += streamDeleteItem(s,&id);
} }
/* Propagate the write if needed. */
if (deleted) {
signalModifiedKey(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STREAM,"xdel",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_STREAM,"xdel",c->argv[1],c->db->id);
server.dirty += deleted; server.dirty += deleted;
}
addReplyLongLong(c,deleted); addReplyLongLong(c,deleted);
} }
...@@ -2100,7 +2355,7 @@ void xdelCommand(client *c) { ...@@ -2100,7 +2355,7 @@ void xdelCommand(client *c) {
* *
* List of options: * List of options:
* *
* MAXLEN [~] <count> -- Trim so that the stream will be capped at * MAXLEN [~|=] <count> -- Trim so that the stream will be capped at
* the specified length. Use ~ before the * the specified length. Use ~ before the
* count in order to demand approximated trimming * count in order to demand approximated trimming
* (like XADD MAXLEN option). * (like XADD MAXLEN option).
...@@ -2119,9 +2374,10 @@ void xtrimCommand(client *c) { ...@@ -2119,9 +2374,10 @@ void xtrimCommand(client *c) {
/* Argument parsing. */ /* Argument parsing. */
int trim_strategy = TRIM_STRATEGY_NONE; int trim_strategy = TRIM_STRATEGY_NONE;
long long maxlen = 0; /* 0 means no maximum length. */ long long maxlen = -1; /* If left to -1 no trimming is performed. */
int approx_maxlen = 0; /* If 1 only delete whole radix tree nodes, so int approx_maxlen = 0; /* If 1 only delete whole radix tree nodes, so
the maxium length is not applied verbatim. */ the maxium length is not applied verbatim. */
int maxlen_arg_idx = 0; /* Index of the count in MAXLEN, for rewriting. */
/* Parse options. */ /* Parse options. */
int i = 2; /* Start of options. */ int i = 2; /* Start of options. */
...@@ -2129,16 +2385,25 @@ void xtrimCommand(client *c) { ...@@ -2129,16 +2385,25 @@ void xtrimCommand(client *c) {
int moreargs = (c->argc-1) - i; /* Number of additional arguments. */ int moreargs = (c->argc-1) - i; /* Number of additional arguments. */
char *opt = c->argv[i]->ptr; char *opt = c->argv[i]->ptr;
if (!strcasecmp(opt,"maxlen") && moreargs) { if (!strcasecmp(opt,"maxlen") && moreargs) {
approx_maxlen = 0;
trim_strategy = TRIM_STRATEGY_MAXLEN; trim_strategy = TRIM_STRATEGY_MAXLEN;
char *next = c->argv[i+1]->ptr; char *next = c->argv[i+1]->ptr;
/* Check for the form MAXLEN ~ <count>. */ /* Check for the form MAXLEN ~ <count>. */
if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') { if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') {
approx_maxlen = 1; approx_maxlen = 1;
i++; i++;
} else if (moreargs >= 2 && next[0] == '=' && next[1] == '\0') {
i++;
} }
if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL) if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL)
!= C_OK) return; != C_OK) return;
if (maxlen < 0) {
addReplyError(c,"The MAXLEN argument must be >= 0.");
return;
}
i++; i++;
maxlen_arg_idx = i;
} else { } else {
addReply(c,shared.syntaxerr); addReply(c,shared.syntaxerr);
return; return;
...@@ -2159,11 +2424,12 @@ void xtrimCommand(client *c) { ...@@ -2159,11 +2424,12 @@ void xtrimCommand(client *c) {
signalModifiedKey(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id);
server.dirty += deleted; server.dirty += deleted;
if (approx_maxlen) streamRewriteApproxMaxlen(c,s,maxlen_arg_idx);
} }
addReplyLongLong(c,deleted); addReplyLongLong(c,deleted);
} }
/* XINFO CONSUMERS key group /* XINFO CONSUMERS <key> <group>
* XINFO GROUPS <key> * XINFO GROUPS <key>
* XINFO STREAM <key> * XINFO STREAM <key>
* XINFO HELP. */ * XINFO HELP. */
...@@ -2209,7 +2475,7 @@ NULL ...@@ -2209,7 +2475,7 @@ NULL
return; return;
} }
addReplyMultiBulkLen(c,raxSize(cg->consumers)); addReplyArrayLen(c,raxSize(cg->consumers));
raxIterator ri; raxIterator ri;
raxStart(&ri,cg->consumers); raxStart(&ri,cg->consumers);
raxSeek(&ri,"^",NULL,0); raxSeek(&ri,"^",NULL,0);
...@@ -2219,48 +2485,52 @@ NULL ...@@ -2219,48 +2485,52 @@ NULL
mstime_t idle = now - consumer->seen_time; mstime_t idle = now - consumer->seen_time;
if (idle < 0) idle = 0; if (idle < 0) idle = 0;
addReplyMultiBulkLen(c,6); addReplyMapLen(c,3);
addReplyStatus(c,"name"); addReplyBulkCString(c,"name");
addReplyBulkCBuffer(c,consumer->name,sdslen(consumer->name)); addReplyBulkCBuffer(c,consumer->name,sdslen(consumer->name));
addReplyStatus(c,"pending"); addReplyBulkCString(c,"pending");
addReplyLongLong(c,raxSize(consumer->pel)); addReplyLongLong(c,raxSize(consumer->pel));
addReplyStatus(c,"idle"); addReplyBulkCString(c,"idle");
addReplyLongLong(c,idle); addReplyLongLong(c,idle);
} }
raxStop(&ri); raxStop(&ri);
} else if (!strcasecmp(opt,"GROUPS") && c->argc == 3) { } else if (!strcasecmp(opt,"GROUPS") && c->argc == 3) {
/* XINFO GROUPS <key>. */ /* XINFO GROUPS <key>. */
if (s->cgroups == NULL) { if (s->cgroups == NULL) {
addReplyMultiBulkLen(c,0); addReplyArrayLen(c,0);
return; return;
} }
addReplyMultiBulkLen(c,raxSize(s->cgroups)); addReplyArrayLen(c,raxSize(s->cgroups));
raxIterator ri; raxIterator ri;
raxStart(&ri,s->cgroups); raxStart(&ri,s->cgroups);
raxSeek(&ri,"^",NULL,0); raxSeek(&ri,"^",NULL,0);
while(raxNext(&ri)) { while(raxNext(&ri)) {
streamCG *cg = ri.data; streamCG *cg = ri.data;
addReplyMultiBulkLen(c,6); addReplyMapLen(c,4);
addReplyStatus(c,"name"); addReplyBulkCString(c,"name");
addReplyBulkCBuffer(c,ri.key,ri.key_len); addReplyBulkCBuffer(c,ri.key,ri.key_len);
addReplyStatus(c,"consumers"); addReplyBulkCString(c,"consumers");
addReplyLongLong(c,raxSize(cg->consumers)); addReplyLongLong(c,raxSize(cg->consumers));
addReplyStatus(c,"pending"); addReplyBulkCString(c,"pending");
addReplyLongLong(c,raxSize(cg->pel)); addReplyLongLong(c,raxSize(cg->pel));
addReplyBulkCString(c,"last-delivered-id");
addReplyStreamID(c,&cg->last_id);
} }
raxStop(&ri); raxStop(&ri);
} else if (!strcasecmp(opt,"STREAM") && c->argc == 3) { } else if (!strcasecmp(opt,"STREAM") && c->argc == 3) {
/* XINFO STREAM <key> (or the alias XINFO <key>). */ /* XINFO STREAM <key> (or the alias XINFO <key>). */
addReplyMultiBulkLen(c,12); addReplyMapLen(c,7);
addReplyStatus(c,"length"); addReplyBulkCString(c,"length");
addReplyLongLong(c,s->length); addReplyLongLong(c,s->length);
addReplyStatus(c,"radix-tree-keys"); addReplyBulkCString(c,"radix-tree-keys");
addReplyLongLong(c,raxSize(s->rax)); addReplyLongLong(c,raxSize(s->rax));
addReplyStatus(c,"radix-tree-nodes"); addReplyBulkCString(c,"radix-tree-nodes");
addReplyLongLong(c,s->rax->numnodes); addReplyLongLong(c,s->rax->numnodes);
addReplyStatus(c,"groups"); addReplyBulkCString(c,"groups");
addReplyLongLong(c,s->cgroups ? raxSize(s->cgroups) : 0); addReplyLongLong(c,s->cgroups ? raxSize(s->cgroups) : 0);
addReplyBulkCString(c,"last-generated-id");
addReplyStreamID(c,&s->last_id);
/* To emit the first/last entry we us the streamReplyWithRange() /* To emit the first/last entry we us the streamReplyWithRange()
* API. */ * API. */
...@@ -2268,16 +2538,16 @@ NULL ...@@ -2268,16 +2538,16 @@ NULL
streamID start, end; streamID start, end;
start.ms = start.seq = 0; start.ms = start.seq = 0;
end.ms = end.seq = UINT64_MAX; end.ms = end.seq = UINT64_MAX;
addReplyStatus(c,"first-entry"); addReplyBulkCString(c,"first-entry");
count = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL, count = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL,
STREAM_RWR_RAWENTRIES,NULL); STREAM_RWR_RAWENTRIES,NULL);
if (!count) addReply(c,shared.nullbulk); if (!count) addReplyNull(c);
addReplyStatus(c,"last-entry"); addReplyBulkCString(c,"last-entry");
count = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL, count = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL,
STREAM_RWR_RAWENTRIES,NULL); STREAM_RWR_RAWENTRIES,NULL);
if (!count) addReply(c,shared.nullbulk); if (!count) addReplyNull(c);
} else { } else {
addReplyError(c,"syntax error, try 'XINFO HELP'"); addReplySubcommandSyntaxError(c);
} }
} }
...@@ -80,7 +80,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire, ...@@ -80,7 +80,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
if ((flags & OBJ_SET_NX && lookupKeyWrite(c->db,key) != NULL) || if ((flags & OBJ_SET_NX && lookupKeyWrite(c->db,key) != NULL) ||
(flags & OBJ_SET_XX && lookupKeyWrite(c->db,key) == NULL)) (flags & OBJ_SET_XX && lookupKeyWrite(c->db,key) == NULL))
{ {
addReply(c, abort_reply ? abort_reply : shared.nullbulk); addReply(c, abort_reply ? abort_reply : shared.null[c->resp]);
return; return;
} }
setKey(c->db,key,val); setKey(c->db,key,val);
...@@ -157,7 +157,7 @@ void psetexCommand(client *c) { ...@@ -157,7 +157,7 @@ void psetexCommand(client *c) {
int getGenericCommand(client *c) { int getGenericCommand(client *c) {
robj *o; robj *o;
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk)) == NULL) if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL)
return C_OK; return C_OK;
if (o->type != OBJ_STRING) { if (o->type != OBJ_STRING) {
...@@ -285,14 +285,14 @@ void getrangeCommand(client *c) { ...@@ -285,14 +285,14 @@ void getrangeCommand(client *c) {
void mgetCommand(client *c) { void mgetCommand(client *c) {
int j; int j;
addReplyMultiBulkLen(c,c->argc-1); addReplyArrayLen(c,c->argc-1);
for (j = 1; j < c->argc; j++) { for (j = 1; j < c->argc; j++) {
robj *o = lookupKeyRead(c->db,c->argv[j]); robj *o = lookupKeyRead(c->db,c->argv[j]);
if (o == NULL) { if (o == NULL) {
addReply(c,shared.nullbulk); addReplyNull(c);
} else { } else {
if (o->type != OBJ_STRING) { if (o->type != OBJ_STRING) {
addReply(c,shared.nullbulk); addReplyNull(c);
} else { } else {
addReplyBulk(c,o); addReplyBulk(c,o);
} }
...@@ -301,25 +301,23 @@ void mgetCommand(client *c) { ...@@ -301,25 +301,23 @@ void mgetCommand(client *c) {
} }
void msetGenericCommand(client *c, int nx) { void msetGenericCommand(client *c, int nx) {
int j, busykeys = 0; int j;
if ((c->argc % 2) == 0) { if ((c->argc % 2) == 0) {
addReplyError(c,"wrong number of arguments for MSET"); addReplyError(c,"wrong number of arguments for MSET");
return; return;
} }
/* Handle the NX flag. The MSETNX semantic is to return zero and don't /* Handle the NX flag. The MSETNX semantic is to return zero and don't
* set nothing at all if at least one already key exists. */ * set anything if at least one key alerady exists. */
if (nx) { if (nx) {
for (j = 1; j < c->argc; j += 2) { for (j = 1; j < c->argc; j += 2) {
if (lookupKeyWrite(c->db,c->argv[j]) != NULL) { if (lookupKeyWrite(c->db,c->argv[j]) != NULL) {
busykeys++;
}
}
if (busykeys) {
addReply(c, shared.czero); addReply(c, shared.czero);
return; return;
} }
} }
}
for (j = 1; j < c->argc; j += 2) { for (j = 1; j < c->argc; j += 2) {
c->argv[j+1] = tryObjectEncoding(c->argv[j+1]); c->argv[j+1] = tryObjectEncoding(c->argv[j+1]);
......
...@@ -244,6 +244,61 @@ int zslDelete(zskiplist *zsl, double score, sds ele, zskiplistNode **node) { ...@@ -244,6 +244,61 @@ int zslDelete(zskiplist *zsl, double score, sds ele, zskiplistNode **node) {
return 0; /* not found */ return 0; /* not found */
} }
/* Update the score of an elmenent inside the sorted set skiplist.
* Note that the element must exist and must match 'score'.
* This function does not update the score in the hash table side, the
* caller should take care of it.
*
* Note that this function attempts to just update the node, in case after
* the score update, the node would be exactly at the same position.
* Otherwise the skiplist is modified by removing and re-adding a new
* element, which is more costly.
*
* The function returns the updated element skiplist node pointer. */
zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) {
zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x;
int i;
/* We need to seek to element to update to start: this is useful anyway,
* we'll have to update or remove it. */
x = zsl->header;
for (i = zsl->level-1; i >= 0; i--) {
while (x->level[i].forward &&
(x->level[i].forward->score < curscore ||
(x->level[i].forward->score == curscore &&
sdscmp(x->level[i].forward->ele,ele) < 0)))
{
x = x->level[i].forward;
}
update[i] = x;
}
/* Jump to our element: note that this function assumes that the
* element with the matching score exists. */
x = x->level[0].forward;
serverAssert(x && curscore == x->score && sdscmp(x->ele,ele) == 0);
/* If the node, after the score update, would be still exactly
* at the same position, we can just update the score without
* actually removing and re-inserting the element in the skiplist. */
if ((x->backward == NULL || x->backward->score < newscore) &&
(x->level[0].forward == NULL || x->level[0].forward->score > newscore))
{
x->score = newscore;
return x;
}
/* No way to reuse the old node: we need to remove and insert a new
* one at a different place. */
zslDeleteNode(zsl, x, update);
zskiplistNode *newnode = zslInsert(zsl,newscore,x->ele);
/* We reused the old node x->ele SDS string, free the node now
* since zslInsert created a new one. */
x->ele = NULL;
zslFreeNode(x);
return newnode;
}
int zslValueGteMin(double value, zrangespec *spec) { int zslValueGteMin(double value, zrangespec *spec) {
return spec->minex ? (value > spec->min) : (value >= spec->min); return spec->minex ? (value > spec->min) : (value >= spec->min);
} }
...@@ -507,7 +562,7 @@ static int zslParseRange(robj *min, robj *max, zrangespec *spec) { ...@@ -507,7 +562,7 @@ static int zslParseRange(robj *min, robj *max, zrangespec *spec) {
* + means the max string possible * + means the max string possible
* *
* If the string is valid the *dest pointer is set to the redis object * If the string is valid the *dest pointer is set to the redis object
* that will be used for the comparision, and ex will be set to 0 or 1 * that will be used for the comparison, and ex will be set to 0 or 1
* respectively if the item is exclusive or inclusive. C_OK will be * respectively if the item is exclusive or inclusive. C_OK will be
* returned. * returned.
* *
...@@ -519,12 +574,12 @@ int zslParseLexRangeItem(robj *item, sds *dest, int *ex) { ...@@ -519,12 +574,12 @@ int zslParseLexRangeItem(robj *item, sds *dest, int *ex) {
switch(c[0]) { switch(c[0]) {
case '+': case '+':
if (c[1] != '\0') return C_ERR; if (c[1] != '\0') return C_ERR;
*ex = 0; *ex = 1;
*dest = shared.maxstring; *dest = shared.maxstring;
return C_OK; return C_OK;
case '-': case '-':
if (c[1] != '\0') return C_ERR; if (c[1] != '\0') return C_ERR;
*ex = 0; *ex = 1;
*dest = shared.minstring; *dest = shared.minstring;
return C_OK; return C_OK;
case '(': case '(':
...@@ -597,9 +652,8 @@ int zslIsInLexRange(zskiplist *zsl, zlexrangespec *range) { ...@@ -597,9 +652,8 @@ int zslIsInLexRange(zskiplist *zsl, zlexrangespec *range) {
zskiplistNode *x; zskiplistNode *x;
/* Test for ranges that will always be empty. */ /* Test for ranges that will always be empty. */
if (sdscmplex(range->min,range->max) > 1 || int cmp = sdscmplex(range->min,range->max);
(sdscmp(range->min,range->max) == 0 && if (cmp > 0 || (cmp == 0 && (range->minex || range->maxex)))
(range->minex || range->maxex)))
return 0; return 0;
x = zsl->tail; x = zsl->tail;
if (x == NULL || !zslLexValueGteMin(x->ele,range)) if (x == NULL || !zslLexValueGteMin(x->ele,range))
...@@ -872,9 +926,8 @@ int zzlIsInLexRange(unsigned char *zl, zlexrangespec *range) { ...@@ -872,9 +926,8 @@ int zzlIsInLexRange(unsigned char *zl, zlexrangespec *range) {
unsigned char *p; unsigned char *p;
/* Test for ranges that will always be empty. */ /* Test for ranges that will always be empty. */
if (sdscmplex(range->min,range->max) > 1 || int cmp = sdscmplex(range->min,range->max);
(sdscmp(range->min,range->max) == 0 && if (cmp > 0 || (cmp == 0 && (range->minex || range->maxex)))
(range->minex || range->maxex)))
return 0; return 0;
p = ziplistIndex(zl,-2); /* Last element. */ p = ziplistIndex(zl,-2); /* Last element. */
...@@ -1304,9 +1357,8 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) { ...@@ -1304,9 +1357,8 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) {
/* Optimize: check if the element is too large or the list /* Optimize: check if the element is too large or the list
* becomes too long *before* executing zzlInsert. */ * becomes too long *before* executing zzlInsert. */
zobj->ptr = zzlInsert(zobj->ptr,ele,score); zobj->ptr = zzlInsert(zobj->ptr,ele,score);
if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries) if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries ||
zsetConvert(zobj,OBJ_ENCODING_SKIPLIST); sdslen(ele) > server.zset_max_ziplist_value)
if (sdslen(ele) > server.zset_max_ziplist_value)
zsetConvert(zobj,OBJ_ENCODING_SKIPLIST); zsetConvert(zobj,OBJ_ENCODING_SKIPLIST);
if (newscore) *newscore = score; if (newscore) *newscore = score;
*flags |= ZADD_ADDED; *flags |= ZADD_ADDED;
...@@ -1341,13 +1393,7 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) { ...@@ -1341,13 +1393,7 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) {
/* Remove and re-insert when score changes. */ /* Remove and re-insert when score changes. */
if (score != curscore) { if (score != curscore) {
zskiplistNode *node; znode = zslUpdateScore(zs->zsl,curscore,ele,score);
serverAssert(zslDelete(zs->zsl,curscore,ele,&node));
znode = zslInsert(zs->zsl,score,node->ele);
/* We reused the node->ele SDS string, free the node now
* since zslInsert created a new one. */
node->ele = NULL;
zslFreeNode(node);
/* Note that we did not removed the original element from /* Note that we did not removed the original element from
* the hash table representing the sorted set, so we just * the hash table representing the sorted set, so we just
* update the score. */ * update the score. */
...@@ -1591,7 +1637,7 @@ reply_to_client: ...@@ -1591,7 +1637,7 @@ reply_to_client:
if (processed) if (processed)
addReplyDouble(c,score); addReplyDouble(c,score);
else else
addReply(c,shared.nullbulk); addReplyNull(c);
} else { /* ZADD. */ } else { /* ZADD. */
addReplyLongLong(c,ch ? added+updated : added); addReplyLongLong(c,ch ? added+updated : added);
} }
...@@ -2380,7 +2426,7 @@ void zrangeGenericCommand(client *c, int reverse) { ...@@ -2380,7 +2426,7 @@ void zrangeGenericCommand(client *c, int reverse) {
return; return;
} }
if ((zobj = lookupKeyReadOrReply(c,key,shared.emptymultibulk)) == NULL if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL
|| checkType(c,zobj,OBJ_ZSET)) return; || checkType(c,zobj,OBJ_ZSET)) return;
/* Sanitize indexes. */ /* Sanitize indexes. */
...@@ -2392,14 +2438,19 @@ void zrangeGenericCommand(client *c, int reverse) { ...@@ -2392,14 +2438,19 @@ void zrangeGenericCommand(client *c, int reverse) {
/* Invariant: start >= 0, so this test will be true when end < 0. /* Invariant: start >= 0, so this test will be true when end < 0.
* The range is empty when start > end or start >= length. */ * The range is empty when start > end or start >= length. */
if (start > end || start >= llen) { if (start > end || start >= llen) {
addReply(c,shared.emptymultibulk); addReplyNull(c);
return; return;
} }
if (end >= llen) end = llen-1; if (end >= llen) end = llen-1;
rangelen = (end-start)+1; rangelen = (end-start)+1;
/* Return the result in form of a multi-bulk reply */ /* Return the result in form of a multi-bulk reply. RESP3 clients
addReplyMultiBulkLen(c, withscores ? (rangelen*2) : rangelen); * will receive sub arrays with score->element, while RESP2 returned
* a flat array. */
if (withscores && c->resp == 2)
addReplyArrayLen(c, rangelen*2);
else
addReplyArrayLen(c, rangelen);
if (zobj->encoding == OBJ_ENCODING_ZIPLIST) { if (zobj->encoding == OBJ_ENCODING_ZIPLIST) {
unsigned char *zl = zobj->ptr; unsigned char *zl = zobj->ptr;
...@@ -2419,13 +2470,13 @@ void zrangeGenericCommand(client *c, int reverse) { ...@@ -2419,13 +2470,13 @@ void zrangeGenericCommand(client *c, int reverse) {
while (rangelen--) { while (rangelen--) {
serverAssertWithInfo(c,zobj,eptr != NULL && sptr != NULL); serverAssertWithInfo(c,zobj,eptr != NULL && sptr != NULL);
serverAssertWithInfo(c,zobj,ziplistGet(eptr,&vstr,&vlen,&vlong)); serverAssertWithInfo(c,zobj,ziplistGet(eptr,&vstr,&vlen,&vlong));
if (withscores && c->resp > 2) addReplyArrayLen(c,2);
if (vstr == NULL) if (vstr == NULL)
addReplyBulkLongLong(c,vlong); addReplyBulkLongLong(c,vlong);
else else
addReplyBulkCBuffer(c,vstr,vlen); addReplyBulkCBuffer(c,vstr,vlen);
if (withscores) addReplyDouble(c,zzlGetScore(sptr));
if (withscores)
addReplyDouble(c,zzlGetScore(sptr));
if (reverse) if (reverse)
zzlPrev(zl,&eptr,&sptr); zzlPrev(zl,&eptr,&sptr);
...@@ -2453,9 +2504,9 @@ void zrangeGenericCommand(client *c, int reverse) { ...@@ -2453,9 +2504,9 @@ void zrangeGenericCommand(client *c, int reverse) {
while(rangelen--) { while(rangelen--) {
serverAssertWithInfo(c,zobj,ln != NULL); serverAssertWithInfo(c,zobj,ln != NULL);
ele = ln->ele; ele = ln->ele;
if (withscores && c->resp > 2) addReplyArrayLen(c,2);
addReplyBulkCBuffer(c,ele,sdslen(ele)); addReplyBulkCBuffer(c,ele,sdslen(ele));
if (withscores) if (withscores) addReplyDouble(c,ln->score);
addReplyDouble(c,ln->score);
ln = reverse ? ln->backward : ln->level[0].forward; ln = reverse ? ln->backward : ln->level[0].forward;
} }
} else { } else {
...@@ -2523,7 +2574,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { ...@@ -2523,7 +2574,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
} }
/* Ok, lookup the key and get the range */ /* Ok, lookup the key and get the range */
if ((zobj = lookupKeyReadOrReply(c,key,shared.emptymultibulk)) == NULL || if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL ||
checkType(c,zobj,OBJ_ZSET)) return; checkType(c,zobj,OBJ_ZSET)) return;
if (zobj->encoding == OBJ_ENCODING_ZIPLIST) { if (zobj->encoding == OBJ_ENCODING_ZIPLIST) {
...@@ -2543,7 +2594,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { ...@@ -2543,7 +2594,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
/* No "first" element in the specified interval. */ /* No "first" element in the specified interval. */
if (eptr == NULL) { if (eptr == NULL) {
addReply(c, shared.emptymultibulk); addReplyNull(c);
return; return;
} }
...@@ -2554,7 +2605,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { ...@@ -2554,7 +2605,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
/* We don't know in advance how many matching elements there are in the /* We don't know in advance how many matching elements there are in the
* list, so we push this object that will represent the multi-bulk * list, so we push this object that will represent the multi-bulk
* length in the output buffer, and will "fix" it later */ * length in the output buffer, and will "fix" it later */
replylen = addDeferredMultiBulkLength(c); replylen = addReplyDeferredLen(c);
/* If there is an offset, just traverse the number of elements without /* If there is an offset, just traverse the number of elements without
* checking the score because that is done in the next loop. */ * checking the score because that is done in the next loop. */
...@@ -2576,19 +2627,18 @@ void genericZrangebyscoreCommand(client *c, int reverse) { ...@@ -2576,19 +2627,18 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
if (!zslValueLteMax(score,&range)) break; if (!zslValueLteMax(score,&range)) break;
} }
/* We know the element exists, so ziplistGet should always succeed */ /* We know the element exists, so ziplistGet should always
* succeed */
serverAssertWithInfo(c,zobj,ziplistGet(eptr,&vstr,&vlen,&vlong)); serverAssertWithInfo(c,zobj,ziplistGet(eptr,&vstr,&vlen,&vlong));
rangelen++; rangelen++;
if (withscores && c->resp > 2) addReplyArrayLen(c,2);
if (vstr == NULL) { if (vstr == NULL) {
addReplyBulkLongLong(c,vlong); addReplyBulkLongLong(c,vlong);
} else { } else {
addReplyBulkCBuffer(c,vstr,vlen); addReplyBulkCBuffer(c,vstr,vlen);
} }
if (withscores) addReplyDouble(c,score);
if (withscores) {
addReplyDouble(c,score);
}
/* Move to next node */ /* Move to next node */
if (reverse) { if (reverse) {
...@@ -2611,14 +2661,14 @@ void genericZrangebyscoreCommand(client *c, int reverse) { ...@@ -2611,14 +2661,14 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
/* No "first" element in the specified interval. */ /* No "first" element in the specified interval. */
if (ln == NULL) { if (ln == NULL) {
addReply(c, shared.emptymultibulk); addReplyNull(c);
return; return;
} }
/* We don't know in advance how many matching elements there are in the /* We don't know in advance how many matching elements there are in the
* list, so we push this object that will represent the multi-bulk * list, so we push this object that will represent the multi-bulk
* length in the output buffer, and will "fix" it later */ * length in the output buffer, and will "fix" it later */
replylen = addDeferredMultiBulkLength(c); replylen = addReplyDeferredLen(c);
/* If there is an offset, just traverse the number of elements without /* If there is an offset, just traverse the number of elements without
* checking the score because that is done in the next loop. */ * checking the score because that is done in the next loop. */
...@@ -2639,11 +2689,9 @@ void genericZrangebyscoreCommand(client *c, int reverse) { ...@@ -2639,11 +2689,9 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
} }
rangelen++; rangelen++;
if (withscores && c->resp > 2) addReplyArrayLen(c,2);
addReplyBulkCBuffer(c,ln->ele,sdslen(ln->ele)); addReplyBulkCBuffer(c,ln->ele,sdslen(ln->ele));
if (withscores) addReplyDouble(c,ln->score);
if (withscores) {
addReplyDouble(c,ln->score);
}
/* Move to next node */ /* Move to next node */
if (reverse) { if (reverse) {
...@@ -2656,11 +2704,8 @@ void genericZrangebyscoreCommand(client *c, int reverse) { ...@@ -2656,11 +2704,8 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
serverPanic("Unknown sorted set encoding"); serverPanic("Unknown sorted set encoding");
} }
if (withscores) { if (withscores && c->resp == 2) rangelen *= 2;
rangelen *= 2; setDeferredArrayLen(c, replylen, rangelen);
}
setDeferredMultiBulkLength(c, replylen, rangelen);
} }
void zrangebyscoreCommand(client *c) { void zrangebyscoreCommand(client *c) {
...@@ -2860,7 +2905,10 @@ void genericZrangebylexCommand(client *c, int reverse) { ...@@ -2860,7 +2905,10 @@ void genericZrangebylexCommand(client *c, int reverse) {
while (remaining) { while (remaining) {
if (remaining >= 3 && !strcasecmp(c->argv[pos]->ptr,"limit")) { if (remaining >= 3 && !strcasecmp(c->argv[pos]->ptr,"limit")) {
if ((getLongFromObjectOrReply(c, c->argv[pos+1], &offset, NULL) != C_OK) || if ((getLongFromObjectOrReply(c, c->argv[pos+1], &offset, NULL) != C_OK) ||
(getLongFromObjectOrReply(c, c->argv[pos+2], &limit, NULL) != C_OK)) return; (getLongFromObjectOrReply(c, c->argv[pos+2], &limit, NULL) != C_OK)) {
zslFreeLexRange(&range);
return;
}
pos += 3; remaining -= 3; pos += 3; remaining -= 3;
} else { } else {
zslFreeLexRange(&range); zslFreeLexRange(&range);
...@@ -2871,7 +2919,7 @@ void genericZrangebylexCommand(client *c, int reverse) { ...@@ -2871,7 +2919,7 @@ void genericZrangebylexCommand(client *c, int reverse) {
} }
/* Ok, lookup the key and get the range */ /* Ok, lookup the key and get the range */
if ((zobj = lookupKeyReadOrReply(c,key,shared.emptymultibulk)) == NULL || if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL ||
checkType(c,zobj,OBJ_ZSET)) checkType(c,zobj,OBJ_ZSET))
{ {
zslFreeLexRange(&range); zslFreeLexRange(&range);
...@@ -2894,7 +2942,7 @@ void genericZrangebylexCommand(client *c, int reverse) { ...@@ -2894,7 +2942,7 @@ void genericZrangebylexCommand(client *c, int reverse) {
/* No "first" element in the specified interval. */ /* No "first" element in the specified interval. */
if (eptr == NULL) { if (eptr == NULL) {
addReply(c, shared.emptymultibulk); addReplyNull(c);
zslFreeLexRange(&range); zslFreeLexRange(&range);
return; return;
} }
...@@ -2906,7 +2954,7 @@ void genericZrangebylexCommand(client *c, int reverse) { ...@@ -2906,7 +2954,7 @@ void genericZrangebylexCommand(client *c, int reverse) {
/* We don't know in advance how many matching elements there are in the /* We don't know in advance how many matching elements there are in the
* list, so we push this object that will represent the multi-bulk * list, so we push this object that will represent the multi-bulk
* length in the output buffer, and will "fix" it later */ * length in the output buffer, and will "fix" it later */
replylen = addDeferredMultiBulkLength(c); replylen = addReplyDeferredLen(c);
/* If there is an offset, just traverse the number of elements without /* If there is an offset, just traverse the number of elements without
* checking the score because that is done in the next loop. */ * checking the score because that is done in the next loop. */
...@@ -2958,7 +3006,7 @@ void genericZrangebylexCommand(client *c, int reverse) { ...@@ -2958,7 +3006,7 @@ void genericZrangebylexCommand(client *c, int reverse) {
/* No "first" element in the specified interval. */ /* No "first" element in the specified interval. */
if (ln == NULL) { if (ln == NULL) {
addReply(c, shared.emptymultibulk); addReplyNull(c);
zslFreeLexRange(&range); zslFreeLexRange(&range);
return; return;
} }
...@@ -2966,7 +3014,7 @@ void genericZrangebylexCommand(client *c, int reverse) { ...@@ -2966,7 +3014,7 @@ void genericZrangebylexCommand(client *c, int reverse) {
/* We don't know in advance how many matching elements there are in the /* We don't know in advance how many matching elements there are in the
* list, so we push this object that will represent the multi-bulk * list, so we push this object that will represent the multi-bulk
* length in the output buffer, and will "fix" it later */ * length in the output buffer, and will "fix" it later */
replylen = addDeferredMultiBulkLength(c); replylen = addReplyDeferredLen(c);
/* If there is an offset, just traverse the number of elements without /* If there is an offset, just traverse the number of elements without
* checking the score because that is done in the next loop. */ * checking the score because that is done in the next loop. */
...@@ -3001,7 +3049,7 @@ void genericZrangebylexCommand(client *c, int reverse) { ...@@ -3001,7 +3049,7 @@ void genericZrangebylexCommand(client *c, int reverse) {
} }
zslFreeLexRange(&range); zslFreeLexRange(&range);
setDeferredMultiBulkLength(c, replylen, rangelen); setDeferredArrayLen(c, replylen, rangelen);
} }
void zrangebylexCommand(client *c) { void zrangebylexCommand(client *c) {
...@@ -3027,11 +3075,11 @@ void zscoreCommand(client *c) { ...@@ -3027,11 +3075,11 @@ void zscoreCommand(client *c) {
robj *zobj; robj *zobj;
double score; double score;
if ((zobj = lookupKeyReadOrReply(c,key,shared.nullbulk)) == NULL || if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL ||
checkType(c,zobj,OBJ_ZSET)) return; checkType(c,zobj,OBJ_ZSET)) return;
if (zsetScore(zobj,c->argv[2]->ptr,&score) == C_ERR) { if (zsetScore(zobj,c->argv[2]->ptr,&score) == C_ERR) {
addReply(c,shared.nullbulk); addReplyNull(c);
} else { } else {
addReplyDouble(c,score); addReplyDouble(c,score);
} }
...@@ -3043,7 +3091,7 @@ void zrankGenericCommand(client *c, int reverse) { ...@@ -3043,7 +3091,7 @@ void zrankGenericCommand(client *c, int reverse) {
robj *zobj; robj *zobj;
long rank; long rank;
if ((zobj = lookupKeyReadOrReply(c,key,shared.nullbulk)) == NULL || if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL ||
checkType(c,zobj,OBJ_ZSET)) return; checkType(c,zobj,OBJ_ZSET)) return;
serverAssertWithInfo(c,ele,sdsEncodedObject(ele)); serverAssertWithInfo(c,ele,sdsEncodedObject(ele));
...@@ -3051,7 +3099,7 @@ void zrankGenericCommand(client *c, int reverse) { ...@@ -3051,7 +3099,7 @@ void zrankGenericCommand(client *c, int reverse) {
if (rank >= 0) { if (rank >= 0) {
addReplyLongLong(c,rank); addReplyLongLong(c,rank);
} else { } else {
addReply(c,shared.nullbulk); addReplyNull(c);
} }
} }
...@@ -3094,7 +3142,10 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey ...@@ -3094,7 +3142,10 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
if (countarg) { if (countarg) {
if (getLongFromObjectOrReply(c,countarg,&count,NULL) != C_OK) if (getLongFromObjectOrReply(c,countarg,&count,NULL) != C_OK)
return; return;
if (count < 0) count = 1; if (count <= 0) {
addReply(c,shared.emptyarray);
return;
}
} }
/* Check type and break on the first error, otherwise identify candidate. */ /* Check type and break on the first error, otherwise identify candidate. */
...@@ -3109,11 +3160,11 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey ...@@ -3109,11 +3160,11 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
/* No candidate for zpopping, return empty. */ /* No candidate for zpopping, return empty. */
if (!zobj) { if (!zobj) {
addReply(c,shared.emptymultibulk); addReplyNull(c);
return; return;
} }
void *arraylen_ptr = addDeferredMultiBulkLength(c); void *arraylen_ptr = addReplyDeferredLen(c);
long arraylen = 0; long arraylen = 0;
/* We emit the key only for the blocking variant. */ /* We emit the key only for the blocking variant. */
...@@ -3180,7 +3231,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey ...@@ -3180,7 +3231,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
} }
} while(--count); } while(--count);
setDeferredMultiBulkLength(c,arraylen_ptr,arraylen + (emitkey != 0)); setDeferredArrayLen(c,arraylen_ptr,arraylen + (emitkey != 0));
} }
/* ZPOPMIN key [<count>] */ /* ZPOPMIN key [<count>] */
...@@ -3235,7 +3286,7 @@ void blockingGenericZpopCommand(client *c, int where) { ...@@ -3235,7 +3286,7 @@ void blockingGenericZpopCommand(client *c, int where) {
/* If we are inside a MULTI/EXEC and the zset is empty the only thing /* If we are inside a MULTI/EXEC and the zset is empty the only thing
* we can do is treating it as a timeout (even with timeout 0). */ * we can do is treating it as a timeout (even with timeout 0). */
if (c->flags & CLIENT_MULTI) { if (c->flags & CLIENT_MULTI) {
addReply(c,shared.nullmultibulk); addReplyNullArray(c);
return; return;
} }
......
/* tracking.c - Client side caching: keys tracking and invalidation
*
* Copyright (c) 2019, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "server.h"
/* The tracking table is constituted by 2^24 radix trees (each tree, and the
* table itself, are allocated in a lazy way only when needed) tracking
* clients that may have certain keys in their local, client side, cache.
*
* Keys are grouped into 2^24 slots, in a way similar to Redis Cluster hash
* slots, however here the function we use is crc64, taking the least
* significant 24 bits of the output.
*
* When a client enables tracking with "CLIENT TRACKING on", each key served to
* the client is hashed to one of such slots, and Redis will remember what
* client may have keys about such slot. Later, when a key in a given slot is
* modified, all the clients that may have local copies of keys in that slot
* will receive an invalidation message. There is no distinction of database
* number: a single table is used.
*
* Clients will normally take frequently requested objects in memory, removing
* them when invalidation messages are received. A strategy clients may use is
* to just cache objects in a dictionary, associating to each cached object
* some incremental epoch, or just a timestamp. When invalidation messages are
* received clients may store, in a different table, the timestamp (or epoch)
* of the invalidation of such given slot: later when accessing objects, the
* eviction of stale objects may be performed in a lazy way by checking if the
* cached object timestamp is older than the invalidation timestamp for such
* objects.
*
* The output of the 24 bit hash function is very large (more than 16 million
* possible slots), so clients that may want to use less resources may only
* use the most significant bits instead of the full 24 bits. */
#define TRACKING_TABLE_SIZE (1<<24)
rax **TrackingTable = NULL;
unsigned long TrackingTableUsedSlots = 0;
robj *TrackingChannelName;
/* Remove the tracking state from the client 'c'. Note that there is not much
* to do for us here, if not to decrement the counter of the clients in
* tracking mode, because we just store the ID of the client in the tracking
* table, so we'll remove the ID reference in a lazy way. Otherwise when a
* client with many entries in the table is removed, it would cost a lot of
* time to do the cleanup. */
void disableTracking(client *c) {
if (c->flags & CLIENT_TRACKING) {
server.tracking_clients--;
c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR);
}
}
/* Enable the tracking state for the client 'c', and as a side effect allocates
* the tracking table if needed. If the 'redirect_to' argument is non zero, the
* invalidation messages for this client will be sent to the client ID
* specified by the 'redirect_to' argument. Note that if such client will
* eventually get freed, we'll send a message to the original client to
* inform it of the condition. Multiple clients can redirect the invalidation
* messages to the same client ID. */
void enableTracking(client *c, uint64_t redirect_to) {
if (c->flags & CLIENT_TRACKING) return;
c->flags |= CLIENT_TRACKING;
c->flags &= ~CLIENT_TRACKING_BROKEN_REDIR;
c->client_tracking_redirection = redirect_to;
server.tracking_clients++;
if (TrackingTable == NULL) {
TrackingTable = zcalloc(sizeof(rax*) * TRACKING_TABLE_SIZE);
TrackingChannelName = createStringObject("__redis__:invalidate",20);
}
}
/* This function is called after the excution of a readonly command in the
* case the client 'c' has keys tracking enabled. It will populate the
* tracking ivalidation table according to the keys the user fetched, so that
* Redis will know what are the clients that should receive an invalidation
* message with certain groups of keys are modified. */
void trackingRememberKeys(client *c) {
int numkeys;
int *keys = getKeysFromCommand(c->cmd,c->argv,c->argc,&numkeys);
if (keys == NULL) return;
for(int j = 0; j < numkeys; j++) {
int idx = keys[j];
sds sdskey = c->argv[idx]->ptr;
uint64_t hash = crc64(0,
(unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1);
if (TrackingTable[hash] == NULL) {
TrackingTable[hash] = raxNew();
TrackingTableUsedSlots++;
}
raxTryInsert(TrackingTable[hash],
(unsigned char*)&c->id,sizeof(c->id),NULL,NULL);
}
getKeysFreeResult(keys);
}
void sendTrackingMessage(client *c, long long hash) {
int using_redirection = 0;
if (c->client_tracking_redirection) {
client *redir = lookupClientByID(c->client_tracking_redirection);
if (!redir) {
/* We need to signal to the original connection that we
* are unable to send invalidation messages to the redirected
* connection, because the client no longer exist. */
if (c->resp > 2) {
addReplyPushLen(c,3);
addReplyBulkCBuffer(c,"tracking-redir-broken",21);
addReplyLongLong(c,c->client_tracking_redirection);
}
return;
}
c = redir;
using_redirection = 1;
}
/* Only send such info for clients in RESP version 3 or more. However
* if redirection is active, and the connection we redirect to is
* in Pub/Sub mode, we can support the feature with RESP 2 as well,
* by sending Pub/Sub messages in the __redis__:invalidate channel. */
if (c->resp > 2) {
addReplyPushLen(c,2);
addReplyBulkCBuffer(c,"invalidate",10);
addReplyLongLong(c,hash);
} else if (using_redirection && c->flags & CLIENT_PUBSUB) {
robj *msg = createStringObjectFromLongLong(hash);
addReplyPubsubMessage(c,TrackingChannelName,msg);
decrRefCount(msg);
}
}
/* Invalidates a caching slot: this is actually the low level implementation
* of the API that Redis calls externally, that is trackingInvalidateKey(). */
void trackingInvalidateSlot(uint64_t slot) {
if (TrackingTable == NULL || TrackingTable[slot] == NULL) return;
raxIterator ri;
raxStart(&ri,TrackingTable[slot]);
raxSeek(&ri,"^",NULL,0);
while(raxNext(&ri)) {
uint64_t id;
memcpy(&id,ri.key,ri.key_len);
client *c = lookupClientByID(id);
if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue;
sendTrackingMessage(c,slot);
}
raxStop(&ri);
/* Free the tracking table: we'll create the radix tree and populate it
* again if more keys will be modified in this caching slot. */
raxFree(TrackingTable[slot]);
TrackingTable[slot] = NULL;
TrackingTableUsedSlots--;
}
/* This function is called from signalModifiedKey() or other places in Redis
* when a key changes value. In the context of keys tracking, our task here is
* to send a notification to every client that may have keys about such caching
* slot. */
void trackingInvalidateKey(robj *keyobj) {
if (TrackingTable == NULL || TrackingTableUsedSlots == 0) return;
sds sdskey = keyobj->ptr;
uint64_t hash = crc64(0,
(unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1);
trackingInvalidateSlot(hash);
}
/* This function is called when one or all the Redis databases are flushed
* (dbid == -1 in case of FLUSHALL). Caching slots are not specific for
* each DB but are global: currently what we do is sending a special
* notification to clients with tracking enabled, invalidating the caching
* slot "-1", which means, "all the keys", in order to avoid flooding clients
* with many invalidation messages for all the keys they may hold.
*
* However trying to flush the tracking table here is very costly:
* we need scanning 16 million caching slots in the table to check
* if they are used, this introduces a big delay. So what we do is to really
* flush the table in the case of FLUSHALL. When a FLUSHDB is called instead
* we just send the invalidation message to all the clients, but don't
* flush the table: it will slowly get garbage collected as more keys
* are modified in the used caching slots. */
void trackingInvalidateKeysOnFlush(int dbid) {
if (server.tracking_clients) {
listNode *ln;
listIter li;
listRewind(server.clients,&li);
while ((ln = listNext(&li)) != NULL) {
client *c = listNodeValue(ln);
if (c->flags & CLIENT_TRACKING) {
sendTrackingMessage(c,-1);
}
}
}
/* In case of FLUSHALL, reclaim all the memory used by tracking. */
if (dbid == -1 && TrackingTable) {
for (int j = 0; j < TRACKING_TABLE_SIZE && TrackingTableUsedSlots > 0; j++) {
if (TrackingTable[j] != NULL) {
raxFree(TrackingTable[j]);
TrackingTable[j] = NULL;
TrackingTableUsedSlots--;
}
}
/* If there are no clients with tracking enabled, we can even
* reclaim the memory used by the table itself. The code assumes
* the table is allocated only if there is at least one client alive
* with tracking enabled. */
if (server.tracking_clients == 0) {
zfree(TrackingTable);
TrackingTable = NULL;
}
}
}
/* Tracking forces Redis to remember information about which client may have
* keys about certian caching slots. In workloads where there are a lot of
* reads, but keys are hardly modified, the amount of information we have
* to remember server side could be a lot: for each 16 millions of caching
* slots we may end with a radix tree containing many entries.
*
* So Redis allows the user to configure a maximum fill rate for the
* invalidation table. This function makes sure that we don't go over the
* specified fill rate: if we are over, we can just evict informations about
* random caching slots, and send invalidation messages to clients like if
* the key was modified. */
void trackingLimitUsedSlots(void) {
static unsigned int timeout_counter = 0;
if (server.tracking_table_max_fill == 0) return; /* No limits set. */
unsigned int max_slots =
(TRACKING_TABLE_SIZE/100) * server.tracking_table_max_fill;
if (TrackingTableUsedSlots <= max_slots) {
timeout_counter = 0;
return; /* Limit not reached. */
}
/* We have to invalidate a few slots to reach the limit again. The effort
* we do here is proportional to the number of times we entered this
* function and found that we are still over the limit. */
int effort = 100 * (timeout_counter+1);
/* Let's start at a random position, and perform linear probing, in order
* to improve cache locality. However once we are able to find an used
* slot, jump again randomly, in order to avoid creating big holes in the
* table (that will make this funciton use more resourced later). */
while(effort > 0) {
unsigned int idx = rand() % TRACKING_TABLE_SIZE;
do {
effort--;
idx = (idx+1) % TRACKING_TABLE_SIZE;
if (TrackingTable[idx] != NULL) {
trackingInvalidateSlot(idx);
if (TrackingTableUsedSlots <= max_slots) {
timeout_counter = 0;
return; /* Return ASAP: we are again under the limit. */
} else {
break; /* Jump to next random position. */
}
}
} while(effort > 0);
}
timeout_counter++;
}
/* This is just used in order to access the amount of used slots in the
* tracking table. */
unsigned long long trackingGetUsedSlots(void) {
return TrackingTableUsedSlots;
}
...@@ -39,6 +39,7 @@ ...@@ -39,6 +39,7 @@
#include <float.h> #include <float.h>
#include <stdint.h> #include <stdint.h>
#include <errno.h> #include <errno.h>
#include <time.h>
#include "util.h" #include "util.h"
#include "sha1.h" #include "sha1.h"
...@@ -47,7 +48,7 @@ ...@@ -47,7 +48,7 @@
int stringmatchlen(const char *pattern, int patternLen, int stringmatchlen(const char *pattern, int patternLen,
const char *string, int stringLen, int nocase) const char *string, int stringLen, int nocase)
{ {
while(patternLen) { while(patternLen && stringLen) {
switch(pattern[0]) { switch(pattern[0]) {
case '*': case '*':
while (pattern[1] == '*') { while (pattern[1] == '*') {
...@@ -170,6 +171,22 @@ int stringmatch(const char *pattern, const char *string, int nocase) { ...@@ -170,6 +171,22 @@ int stringmatch(const char *pattern, const char *string, int nocase) {
return stringmatchlen(pattern,strlen(pattern),string,strlen(string),nocase); return stringmatchlen(pattern,strlen(pattern),string,strlen(string),nocase);
} }
/* Fuzz stringmatchlen() trying to crash it with bad input. */
int stringmatchlen_fuzz_test(void) {
char str[32];
char pat[32];
int cycles = 10000000;
int total_matches = 0;
while(cycles--) {
int strlen = rand() % sizeof(str);
int patlen = rand() % sizeof(pat);
for (int j = 0; j < strlen; j++) str[j] = rand() % 128;
for (int j = 0; j < patlen; j++) pat[j] = rand() % 128;
total_matches += stringmatchlen(pat, patlen, str, strlen, 0);
}
return total_matches;
}
/* Convert a string representing an amount of memory into the number of /* Convert a string representing an amount of memory into the number of
* bytes, so for instance memtoll("1Gb") will return 1073741824 that is * bytes, so for instance memtoll("1Gb") will return 1073741824 that is
* (1024*1024*1024). * (1024*1024*1024).
...@@ -346,6 +363,7 @@ int string2ll(const char *s, size_t slen, long long *value) { ...@@ -346,6 +363,7 @@ int string2ll(const char *s, size_t slen, long long *value) {
int negative = 0; int negative = 0;
unsigned long long v; unsigned long long v;
/* A zero length string is not a valid number. */
if (plen == slen) if (plen == slen)
return 0; return 0;
...@@ -355,6 +373,8 @@ int string2ll(const char *s, size_t slen, long long *value) { ...@@ -355,6 +373,8 @@ int string2ll(const char *s, size_t slen, long long *value) {
return 1; return 1;
} }
/* Handle negative numbers: just set a flag and continue like if it
* was a positive number. Later convert into negative. */
if (p[0] == '-') { if (p[0] == '-') {
negative = 1; negative = 1;
p++; plen++; p++; plen++;
...@@ -368,13 +388,11 @@ int string2ll(const char *s, size_t slen, long long *value) { ...@@ -368,13 +388,11 @@ int string2ll(const char *s, size_t slen, long long *value) {
if (p[0] >= '1' && p[0] <= '9') { if (p[0] >= '1' && p[0] <= '9') {
v = p[0]-'0'; v = p[0]-'0';
p++; plen++; p++; plen++;
} else if (p[0] == '0' && slen == 1) {
*value = 0;
return 1;
} else { } else {
return 0; return 0;
} }
/* Parse all the other digits, checking for overflow at every step. */
while (plen < slen && p[0] >= '0' && p[0] <= '9') { while (plen < slen && p[0] >= '0' && p[0] <= '9') {
if (v > (ULLONG_MAX / 10)) /* Overflow. */ if (v > (ULLONG_MAX / 10)) /* Overflow. */
return 0; return 0;
...@@ -391,6 +409,8 @@ int string2ll(const char *s, size_t slen, long long *value) { ...@@ -391,6 +409,8 @@ int string2ll(const char *s, size_t slen, long long *value) {
if (plen < slen) if (plen < slen)
return 0; return 0;
/* Convert to negative if needed, and do the final overflow check when
* converting from unsigned long long to long long. */
if (negative) { if (negative) {
if (v > ((unsigned long long)(-(LLONG_MIN+1))+1)) /* Overflow. */ if (v > ((unsigned long long)(-(LLONG_MIN+1))+1)) /* Overflow. */
return 0; return 0;
...@@ -427,7 +447,7 @@ int string2l(const char *s, size_t slen, long *lval) { ...@@ -427,7 +447,7 @@ int string2l(const char *s, size_t slen, long *lval) {
* a double: no spaces or other characters before or after the string * a double: no spaces or other characters before or after the string
* representing the number are accepted. */ * representing the number are accepted. */
int string2ld(const char *s, size_t slen, long double *dp) { int string2ld(const char *s, size_t slen, long double *dp) {
char buf[256]; char buf[MAX_LONG_DOUBLE_CHARS];
long double value; long double value;
char *eptr; char *eptr;
...@@ -451,7 +471,7 @@ int string2ld(const char *s, size_t slen, long double *dp) { ...@@ -451,7 +471,7 @@ int string2ld(const char *s, size_t slen, long double *dp) {
/* Convert a double to a string representation. Returns the number of bytes /* Convert a double to a string representation. Returns the number of bytes
* required. The representation should always be parsable by strtod(3). * required. The representation should always be parsable by strtod(3).
* This function does not support human-friendly formatting like ld2string * This function does not support human-friendly formatting like ld2string
* does. It is intented mainly to be used inside t_zset.c when writing scores * does. It is intended mainly to be used inside t_zset.c when writing scores
* into a ziplist representing a sorted set. */ * into a ziplist representing a sorted set. */
int d2string(char *buf, size_t len, double value) { int d2string(char *buf, size_t len, double value) {
if (isnan(value)) { if (isnan(value)) {
...@@ -602,7 +622,7 @@ void getRandomHexChars(char *p, size_t len) { ...@@ -602,7 +622,7 @@ void getRandomHexChars(char *p, size_t len) {
* already, this will be detected and handled correctly. * already, this will be detected and handled correctly.
* *
* The function does not try to normalize everything, but only the obvious * The function does not try to normalize everything, but only the obvious
* case of one or more "../" appearning at the start of "filename" * case of one or more "../" appearing at the start of "filename"
* relative path. */ * relative path. */
sds getAbsolutePath(char *filename) { sds getAbsolutePath(char *filename) {
char cwd[1024]; char cwd[1024];
...@@ -649,6 +669,24 @@ sds getAbsolutePath(char *filename) { ...@@ -649,6 +669,24 @@ sds getAbsolutePath(char *filename) {
return abspath; return abspath;
} }
/*
* Gets the proper timezone in a more portable fashion
* i.e timezone variables are linux specific.
*/
unsigned long getTimeZone(void) {
#ifdef __linux__
return timezone;
#else
struct timeval tv;
struct timezone tz;
gettimeofday(&tv, &tz);
return tz.tz_minuteswest * 60UL;
#endif
}
/* Return true if the specified path is just a file basename without any /* Return true if the specified path is just a file basename without any
* relative or absolute path. This function just checks that no / or \ * relative or absolute path. This function just checks that no / or \
* character exists inside the specified path, that's enough in the * character exists inside the specified path, that's enough in the
......
...@@ -40,6 +40,7 @@ ...@@ -40,6 +40,7 @@
int stringmatchlen(const char *p, int plen, const char *s, int slen, int nocase); int stringmatchlen(const char *p, int plen, const char *s, int slen, int nocase);
int stringmatch(const char *p, const char *s, int nocase); int stringmatch(const char *p, const char *s, int nocase);
int stringmatchlen_fuzz_test(void);
long long memtoll(const char *p, int *err); long long memtoll(const char *p, int *err);
uint32_t digits10(uint64_t v); uint32_t digits10(uint64_t v);
uint32_t sdigits10(int64_t v); uint32_t sdigits10(int64_t v);
...@@ -50,6 +51,7 @@ int string2ld(const char *s, size_t slen, long double *dp); ...@@ -50,6 +51,7 @@ int string2ld(const char *s, size_t slen, long double *dp);
int d2string(char *buf, size_t len, double value); int d2string(char *buf, size_t len, double value);
int ld2string(char *buf, size_t len, long double value, int humanfriendly); int ld2string(char *buf, size_t len, long double value, int humanfriendly);
sds getAbsolutePath(char *filename); sds getAbsolutePath(char *filename);
unsigned long getTimeZone(void);
int pathIsBaseName(char *path); int pathIsBaseName(char *path);
#ifdef REDIS_TEST #ifdef REDIS_TEST
......
...@@ -269,7 +269,7 @@ ...@@ -269,7 +269,7 @@
* Note that this is not how the data is actually encoded, is just what we * Note that this is not how the data is actually encoded, is just what we
* get filled by a function in order to operate more easily. */ * get filled by a function in order to operate more easily. */
typedef struct zlentry { typedef struct zlentry {
unsigned int prevrawlensize; /* Bytes used to encode the previos entry len*/ unsigned int prevrawlensize; /* Bytes used to encode the previous entry len*/
unsigned int prevrawlen; /* Previous entry len. */ unsigned int prevrawlen; /* Previous entry len. */
unsigned int lensize; /* Bytes used to encode this entry type/len. unsigned int lensize; /* Bytes used to encode this entry type/len.
For example strings have a 1, 2 or 5 bytes For example strings have a 1, 2 or 5 bytes
...@@ -431,7 +431,7 @@ unsigned int zipStorePrevEntryLength(unsigned char *p, unsigned int len) { ...@@ -431,7 +431,7 @@ unsigned int zipStorePrevEntryLength(unsigned char *p, unsigned int len) {
/* Return the length of the previous element, and the number of bytes that /* Return the length of the previous element, and the number of bytes that
* are used in order to encode the previous element length. * are used in order to encode the previous element length.
* 'ptr' must point to the prevlen prefix of an entry (that encodes the * 'ptr' must point to the prevlen prefix of an entry (that encodes the
* length of the previos entry in order to navigate the elements backward). * length of the previous entry in order to navigate the elements backward).
* The length of the previous entry is stored in 'prevlen', the number of * The length of the previous entry is stored in 'prevlen', the number of
* bytes needed to encode the previous entry length are stored in * bytes needed to encode the previous entry length are stored in
* 'prevlensize'. */ * 'prevlensize'. */
...@@ -576,7 +576,7 @@ void zipEntry(unsigned char *p, zlentry *e) { ...@@ -576,7 +576,7 @@ void zipEntry(unsigned char *p, zlentry *e) {
/* Create a new empty ziplist. */ /* Create a new empty ziplist. */
unsigned char *ziplistNew(void) { unsigned char *ziplistNew(void) {
unsigned int bytes = ZIPLIST_HEADER_SIZE+1; unsigned int bytes = ZIPLIST_HEADER_SIZE+ZIPLIST_END_SIZE;
unsigned char *zl = zmalloc(bytes); unsigned char *zl = zmalloc(bytes);
ZIPLIST_BYTES(zl) = intrev32ifbe(bytes); ZIPLIST_BYTES(zl) = intrev32ifbe(bytes);
ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(ZIPLIST_HEADER_SIZE); ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(ZIPLIST_HEADER_SIZE);
......
...@@ -148,6 +148,10 @@ void *zrealloc(void *ptr, size_t size) { ...@@ -148,6 +148,10 @@ void *zrealloc(void *ptr, size_t size) {
size_t oldsize; size_t oldsize;
void *newptr; void *newptr;
if (size == 0 && ptr != NULL) {
zfree(ptr);
return NULL;
}
if (ptr == NULL) return zmalloc(size); if (ptr == NULL) return zmalloc(size);
#ifdef HAVE_MALLOC_SIZE #ifdef HAVE_MALLOC_SIZE
oldsize = zmalloc_size(ptr); oldsize = zmalloc_size(ptr);
...@@ -164,7 +168,7 @@ void *zrealloc(void *ptr, size_t size) { ...@@ -164,7 +168,7 @@ void *zrealloc(void *ptr, size_t size) {
if (!newptr) zmalloc_oom_handler(size); if (!newptr) zmalloc_oom_handler(size);
*((size_t*)newptr) = size; *((size_t*)newptr) = size;
update_zmalloc_stat_free(oldsize); update_zmalloc_stat_free(oldsize+PREFIX_SIZE);
update_zmalloc_stat_alloc(size+PREFIX_SIZE); update_zmalloc_stat_alloc(size+PREFIX_SIZE);
return (char*)newptr+PREFIX_SIZE; return (char*)newptr+PREFIX_SIZE;
#endif #endif
...@@ -182,6 +186,9 @@ size_t zmalloc_size(void *ptr) { ...@@ -182,6 +186,9 @@ size_t zmalloc_size(void *ptr) {
if (size&(sizeof(long)-1)) size += sizeof(long)-(size&(sizeof(long)-1)); if (size&(sizeof(long)-1)) size += sizeof(long)-(size&(sizeof(long)-1));
return size+PREFIX_SIZE; return size+PREFIX_SIZE;
} }
size_t zmalloc_usable(void *ptr) {
return zmalloc_size(ptr)-PREFIX_SIZE;
}
#endif #endif
void zfree(void *ptr) { void zfree(void *ptr) {
...@@ -287,6 +294,26 @@ size_t zmalloc_get_rss(void) { ...@@ -287,6 +294,26 @@ size_t zmalloc_get_rss(void) {
return t_info.resident_size; return t_info.resident_size;
} }
#elif defined(__FreeBSD__)
#include <sys/types.h>
#include <sys/sysctl.h>
#include <sys/user.h>
#include <unistd.h>
size_t zmalloc_get_rss(void) {
struct kinfo_proc info;
size_t infolen = sizeof(info);
int mib[4];
mib[0] = CTL_KERN;
mib[1] = KERN_PROC;
mib[2] = KERN_PROC_PID;
mib[3] = getpid();
if (sysctl(mib, 4, &info, &infolen, NULL, 0) == 0)
return (size_t)info.ki_rssize;
return 0L;
}
#else #else
size_t zmalloc_get_rss(void) { size_t zmalloc_get_rss(void) {
/* If we can't get the RSS in an OS-specific way for this system just /* If we can't get the RSS in an OS-specific way for this system just
...@@ -380,7 +407,7 @@ size_t zmalloc_get_private_dirty(long pid) { ...@@ -380,7 +407,7 @@ size_t zmalloc_get_private_dirty(long pid) {
} }
/* Returns the size of physical memory (RAM) in bytes. /* Returns the size of physical memory (RAM) in bytes.
* It looks ugly, but this is the cleanest way to achive cross platform results. * It looks ugly, but this is the cleanest way to achieve cross platform results.
* Cleaned up from: * Cleaned up from:
* *
* http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system * http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system
...@@ -435,4 +462,20 @@ size_t zmalloc_get_memory_size(void) { ...@@ -435,4 +462,20 @@ size_t zmalloc_get_memory_size(void) {
#endif #endif
} }
#ifdef REDIS_TEST
#define UNUSED(x) ((void)(x))
int zmalloc_test(int argc, char **argv) {
void *ptr;
UNUSED(argc);
UNUSED(argv);
printf("Initial used memory: %zu\n", zmalloc_used_memory());
ptr = zmalloc(123);
printf("Allocated 123 bytes; used: %zu\n", zmalloc_used_memory());
ptr = zrealloc(ptr, 456);
printf("Reallocated to 456 bytes; used: %zu\n", zmalloc_used_memory());
zfree(ptr);
printf("Freed pointer; used: %zu\n", zmalloc_used_memory());
return 0;
}
#endif
...@@ -98,6 +98,13 @@ void *zmalloc_no_tcache(size_t size); ...@@ -98,6 +98,13 @@ void *zmalloc_no_tcache(size_t size);
#ifndef HAVE_MALLOC_SIZE #ifndef HAVE_MALLOC_SIZE
size_t zmalloc_size(void *ptr); size_t zmalloc_size(void *ptr);
size_t zmalloc_usable(void *ptr);
#else
#define zmalloc_usable(p) zmalloc_size(p)
#endif
#ifdef REDIS_TEST
int zmalloc_test(int argc, char **argv);
#endif #endif
#endif /* __ZMALLOC_H */ #endif /* __ZMALLOC_H */
...@@ -92,3 +92,80 @@ test "Node #10 should eventually replicate node #5" { ...@@ -92,3 +92,80 @@ test "Node #10 should eventually replicate node #5" {
fail "#10 didn't became slave of #5" fail "#10 didn't became slave of #5"
} }
} }
source "../tests/includes/init-tests.tcl"
# Create a cluster with 3 master and 15 slaves, so that we have 5
# slaves for eatch master.
test "Create a 3 nodes cluster" {
create_cluster 3 15
}
test "Cluster is up" {
assert_cluster_state ok
}
test "The first master has actually 5 slaves" {
assert {[llength [lindex [R 0 role] 2]] == 5}
}
test {Slaves of #0 are instance #3, #6, #9, #12 and #15 as expected} {
set port0 [get_instance_attrib redis 0 port]
assert {[lindex [R 3 role] 2] == $port0}
assert {[lindex [R 6 role] 2] == $port0}
assert {[lindex [R 9 role] 2] == $port0}
assert {[lindex [R 12 role] 2] == $port0}
assert {[lindex [R 15 role] 2] == $port0}
}
test {Instance #3, #6, #9, #12 and #15 synced with the master} {
wait_for_condition 1000 50 {
[RI 3 master_link_status] eq {up} &&
[RI 6 master_link_status] eq {up} &&
[RI 9 master_link_status] eq {up} &&
[RI 12 master_link_status] eq {up} &&
[RI 15 master_link_status] eq {up}
} else {
fail "Instance #3 or #6 or #9 or #12 or #15 master link status is not up"
}
}
proc master_detected {instances} {
foreach instance [dict keys $instances] {
if {[RI $instance role] eq {master}} {
return true
}
}
return false
}
test "New Master down consecutively" {
set instances [dict create 0 1 3 1 6 1 9 1 12 1 15 1]
set loops [expr {[dict size $instances]-1}]
for {set i 0} {$i < $loops} {incr i} {
set master_id -1
foreach instance [dict keys $instances] {
if {[RI $instance role] eq {master}} {
set master_id $instance
break;
}
}
if {$master_id eq -1} {
fail "no master detected, #loop $i"
}
set instances [dict remove $instances $master_id]
kill_instance redis $master_id
wait_for_condition 1000 50 {
[master_detected $instances]
} else {
failover "No failover detected when master $master_id fails"
}
assert_cluster_state ok
}
}
...@@ -49,7 +49,7 @@ start_server {tags {"repl"}} { ...@@ -49,7 +49,7 @@ start_server {tags {"repl"}} {
set fd [open /tmp/repldump2.txt w] set fd [open /tmp/repldump2.txt w]
puts -nonewline $fd $csv2 puts -nonewline $fd $csv2
close $fd close $fd
puts "Master - Slave inconsistency" puts "Master - Replica inconsistency"
puts "Run diff -u against /tmp/repldump*.txt for more info" puts "Run diff -u against /tmp/repldump*.txt for more info"
} }
assert_equal [r debug digest] [r -1 debug digest] assert_equal [r debug digest] [r -1 debug digest]
......
...@@ -29,7 +29,7 @@ start_server {} { ...@@ -29,7 +29,7 @@ start_server {} {
wait_for_condition 50 1000 { wait_for_condition 50 1000 {
[$R(1) dbsize] == 1 && [$R(2) dbsize] == 1 [$R(1) dbsize] == 1 && [$R(2) dbsize] == 1
} else { } else {
fail "Slaves not replicating from master" fail "Replicas not replicating from master"
} }
$R(0) config set repl-backlog-size 10mb $R(0) config set repl-backlog-size 10mb
$R(1) config set repl-backlog-size 10mb $R(1) config set repl-backlog-size 10mb
...@@ -41,12 +41,12 @@ start_server {} { ...@@ -41,12 +41,12 @@ start_server {} {
set elapsed [expr {[clock milliseconds]-$cycle_start_time}] set elapsed [expr {[clock milliseconds]-$cycle_start_time}]
if {$elapsed > $duration*1000} break if {$elapsed > $duration*1000} break
if {rand() < .05} { if {rand() < .05} {
test "PSYNC2 #3899 regression: kill first slave" { test "PSYNC2 #3899 regression: kill first replica" {
$R(1) client kill type master $R(1) client kill type master
} }
} }
if {rand() < .05} { if {rand() < .05} {
test "PSYNC2 #3899 regression: kill chained slave" { test "PSYNC2 #3899 regression: kill chained replica" {
$R(2) client kill type master $R(2) client kill type master
} }
} }
......
...@@ -33,9 +33,8 @@ start_server {} { ...@@ -33,9 +33,8 @@ start_server {} {
set cycle 1 set cycle 1
while {([clock seconds]-$start_time) < $duration} { while {([clock seconds]-$start_time) < $duration} {
test "PSYNC2: --- CYCLE $cycle ---" { test "PSYNC2: --- CYCLE $cycle ---" {}
incr cycle incr cycle
}
# Create a random replication layout. # Create a random replication layout.
# Start with switching master (this simulates a failover). # Start with switching master (this simulates a failover).
...@@ -96,7 +95,7 @@ start_server {} { ...@@ -96,7 +95,7 @@ start_server {} {
if {$disconnect} { if {$disconnect} {
$R($slave_id) client kill type master $R($slave_id) client kill type master
if {$debug_msg} { if {$debug_msg} {
puts "+++ Breaking link for slave #$slave_id" puts "+++ Breaking link for replica #$slave_id"
} }
} }
} }
...@@ -139,6 +138,11 @@ start_server {} { ...@@ -139,6 +138,11 @@ start_server {} {
} }
assert {$sum == 4} assert {$sum == 4}
} }
# Limit anyway the maximum number of cycles. This is useful when the
# test is skipped via --only option of the test suite. In that case
# we don't want to see many seconds of this test being just skipped.
if {$cycle > 50} break
} }
test "PSYNC2: Bring the master back again for next test" { test "PSYNC2: Bring the master back again for next test" {
...@@ -154,7 +158,7 @@ start_server {} { ...@@ -154,7 +158,7 @@ start_server {} {
wait_for_condition 50 1000 { wait_for_condition 50 1000 {
[status $R($master_id) connected_slaves] == 4 [status $R($master_id) connected_slaves] == 4
} else { } else {
fail "Slave not reconnecting" fail "Replica not reconnecting"
} }
} }
...@@ -162,20 +166,23 @@ start_server {} { ...@@ -162,20 +166,23 @@ start_server {} {
# Pick a random slave # Pick a random slave
set slave_id [expr {($master_id+1)%5}] set slave_id [expr {($master_id+1)%5}]
set sync_count [status $R($master_id) sync_full] set sync_count [status $R($master_id) sync_full]
set sync_partial [status $R($master_id) sync_partial_ok]
catch { catch {
$R($slave_id) config rewrite $R($slave_id) config rewrite
$R($slave_id) debug restart $R($slave_id) debug restart
} }
# note: just waiting for connected_slaves==4 has a race condition since
# we might do the check before the master realized that the slave disconnected
wait_for_condition 50 1000 { wait_for_condition 50 1000 {
[status $R($master_id) connected_slaves] == 4 [status $R($master_id) sync_partial_ok] == $sync_partial + 1
} else { } else {
fail "Slave not reconnecting" fail "Replica not reconnecting"
} }
set new_sync_count [status $R($master_id) sync_full] set new_sync_count [status $R($master_id) sync_full]
assert {$sync_count == $new_sync_count} assert {$sync_count == $new_sync_count}
} }
test "PSYNC2: Slave RDB restart with EVALSHA in backlog issue #4483" { test "PSYNC2: Replica RDB restart with EVALSHA in backlog issue #4483" {
# Pick a random slave # Pick a random slave
set slave_id [expr {($master_id+1)%5}] set slave_id [expr {($master_id+1)%5}]
set sync_count [status $R($master_id) sync_full] set sync_count [status $R($master_id) sync_full]
...@@ -190,7 +197,7 @@ start_server {} { ...@@ -190,7 +197,7 @@ start_server {} {
wait_for_condition 50 1000 { wait_for_condition 50 1000 {
[$R($master_id) debug digest] == [$R($slave_id) debug digest] [$R($master_id) debug digest] == [$R($slave_id) debug digest]
} else { } else {
fail "Slave not reconnecting" fail "Replica not reconnecting"
} }
# Prevent the slave from receiving master updates, and at # Prevent the slave from receiving master updates, and at
...@@ -224,7 +231,7 @@ start_server {} { ...@@ -224,7 +231,7 @@ start_server {} {
wait_for_condition 50 1000 { wait_for_condition 50 1000 {
[status $R($master_id) connected_slaves] == 4 [status $R($master_id) connected_slaves] == 4
} else { } else {
fail "Slave not reconnecting" fail "Replica not reconnecting"
} }
set new_sync_count [status $R($master_id) sync_full] set new_sync_count [status $R($master_id) sync_full]
assert {$sync_count == $new_sync_count} assert {$sync_count == $new_sync_count}
...@@ -234,7 +241,7 @@ start_server {} { ...@@ -234,7 +241,7 @@ start_server {} {
wait_for_condition 50 1000 { wait_for_condition 50 1000 {
[$R($master_id) debug digest] == [$R($slave_id) debug digest] [$R($master_id) debug digest] == [$R($slave_id) debug digest]
} else { } else {
fail "Debug digest mismatch between master and slave in post-restart handshake" fail "Debug digest mismatch between master and replica in post-restart handshake"
} }
} }
......
...@@ -48,6 +48,8 @@ start_server [list overrides [list "dir" $server_path]] { ...@@ -48,6 +48,8 @@ start_server [list overrides [list "dir" $server_path]] {
r xadd stream * bar $j r xadd stream * bar $j
} }
} }
r xgroup create stream mygroup 0
r xreadgroup GROUP mygroup Alice COUNT 1 STREAMS stream >
set digest [r debug digest] set digest [r debug digest]
r debug reload r debug reload
set newdigest [r debug digest] set newdigest [r debug digest]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment