Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
56ce42fa
Commit
56ce42fa
authored
Mar 10, 2011
by
Pieter Noordhuis
Browse files
Generic iterator code for usage in ZUNIONSTORE/ZINTERSTORE
parent
bbfe232f
Changes
1
Show whitespace changes
Inline
Side-by-side
src/t_zset.c
View file @
56ce42fa
...
@@ -810,6 +810,12 @@ void zsConvert(robj *zobj, int encoding) {
...
@@ -810,6 +810,12 @@ void zsConvert(robj *zobj, int encoding) {
* Sorted set commands
* Sorted set commands
*----------------------------------------------------------------------------*/
*----------------------------------------------------------------------------*/
// if (zobj->encoding == REDIS_ENCODING_ZIPLIST) {
// } else if (zobj->encoding == REDIS_ENCODING_RAW) {
// } else {
// redisPanic("Unknown sorted set encoding");
// }
/* This generic command implements both ZADD and ZINCRBY. */
/* This generic command implements both ZADD and ZINCRBY. */
void zaddGenericCommand(redisClient *c, int incr) {
void zaddGenericCommand(redisClient *c, int incr) {
static char *nanerr = "resulting score is not a number (NaN)";
static char *nanerr = "resulting score is not a number (NaN)";
...
@@ -1075,16 +1081,327 @@ void zremrangebyrankCommand(redisClient *c) {
...
@@ -1075,16 +1081,327 @@ void zremrangebyrankCommand(redisClient *c) {
}
}
typedef struct {
typedef struct {
dict *dict;
robj *subject;
int type; /* Set, sorted set */
int encoding;
double weight;
double weight;
union {
/* Set iterators. */
union _iterset {
struct {
intset *is;
int ii;
} is;
struct {
dict *dict;
dictIterator *di;
dictEntry *de;
} ht;
} set;
/* Sorted set iterators. */
union _iterzset {
struct {
unsigned char *zl;
unsigned char *eptr, *sptr;
} zl;
struct {
zset *zs;
zskiplistNode *node;
} sl;
} zset;
} iter;
} zsetopsrc;
} zsetopsrc;
int qsortCompareZsetopsrcByCardinality(const void *s1, const void *s2) {
zsetopsrc *d1 = (void*) s1, *d2 = (void*) s2;
/* Use dirty flags for pointers that need to be cleaned up in the next
unsigned long size1, size2;
* iteration over the zsetopval. The dirty flag for the long long value is
size1 = d1->dict ? dictSize(d1->dict) : 0;
* special, since long long values don't need cleanup. Instead, it means that
size2 = d2->dict ? dictSize(d2->dict) : 0;
* we already checked that "ell" holds a long long, or tried to convert another
return size1 - size2;
* representation into a long long value. When this was successful,
* OPVAL_VALID_LL is set as well. */
#define OPVAL_DIRTY_ROBJ 1
#define OPVAL_DIRTY_LL 2
#define OPVAL_VALID_LL 4
/* Store value retrieved from the iterator. */
typedef struct {
int flags;
unsigned char _buf[32]; /* Private buffer. */
robj *ele;
unsigned char *estr;
unsigned int elen;
long long ell;
double score;
} zsetopval;
typedef union _iterset iterset;
typedef union _iterzset iterzset;
void zuiInitIterator(zsetopsrc *op) {
if (op->subject == NULL)
return;
if (op->type == REDIS_SET) {
iterset *it = &op->iter.set;
if (op->encoding == REDIS_ENCODING_INTSET) {
it->is.is = op->subject->ptr;
it->is.ii = 0;
} else if (op->encoding == REDIS_ENCODING_HT) {
it->ht.dict = op->subject->ptr;
it->ht.di = dictGetIterator(op->subject->ptr);
it->ht.de = dictNext(it->ht.di);
} else {
redisPanic("Unknown set encoding");
}
} else if (op->type == REDIS_ZSET) {
iterzset *it = &op->iter.zset;
if (op->encoding == REDIS_ENCODING_ZIPLIST) {
it->zl.zl = op->subject->ptr;
it->zl.eptr = ziplistIndex(it->zl.zl,0);
if (it->zl.eptr != NULL) {
it->zl.sptr = ziplistNext(it->zl.zl,it->zl.eptr);
redisAssert(it->zl.sptr != NULL);
}
} else if (op->encoding == REDIS_ENCODING_RAW) {
it->sl.zs = op->subject->ptr;
it->sl.node = it->sl.zs->zsl->header->level[0].forward;
} else {
redisPanic("Unknown sorted set encoding");
}
} else {
redisPanic("Unsupported type");
}
}
void zuiClearIterator(zsetopsrc *op) {
if (op->subject == NULL)
return;
if (op->type == REDIS_SET) {
iterset *it = &op->iter.set;
if (op->encoding == REDIS_ENCODING_INTSET) {
REDIS_NOTUSED(it); /* skip */
} else if (op->encoding == REDIS_ENCODING_HT) {
dictReleaseIterator(it->ht.di);
} else {
redisPanic("Unknown set encoding");
}
} else if (op->type == REDIS_ZSET) {
iterzset *it = &op->iter.zset;
if (op->encoding == REDIS_ENCODING_ZIPLIST) {
REDIS_NOTUSED(it); /* skip */
} else if (op->encoding == REDIS_ENCODING_RAW) {
REDIS_NOTUSED(it); /* skip */
} else {
redisPanic("Unknown sorted set encoding");
}
} else {
redisPanic("Unsupported type");
}
}
int zuiLength(zsetopsrc *op) {
if (op->subject == NULL)
return 0;
if (op->type == REDIS_SET) {
iterset *it = &op->iter.set;
if (op->encoding == REDIS_ENCODING_INTSET) {
return intsetLen(it->is.is);
} else if (op->encoding == REDIS_ENCODING_HT) {
return dictSize(it->ht.dict);
} else {
redisPanic("Unknown set encoding");
}
} else if (op->type == REDIS_ZSET) {
iterzset *it = &op->iter.zset;
if (op->encoding == REDIS_ENCODING_ZIPLIST) {
return zzlLength(it->zl.zl);
} else if (op->encoding == REDIS_ENCODING_RAW) {
return it->sl.zs->zsl->length;
} else {
redisPanic("Unknown sorted set encoding");
}
} else {
redisPanic("Unsupported type");
}
}
/* Check if the current value is valid. If so, store it in the passed structure
* and move to the next element. If not valid, this means we have reached the
* end of the structure and can abort. */
int zuiNext(zsetopsrc *op, zsetopval *val) {
if (op->subject == NULL)
return 0;
if (val->flags & OPVAL_DIRTY_ROBJ)
decrRefCount(val->ele);
bzero(val,sizeof(zsetopval));
if (op->type == REDIS_SET) {
iterset *it = &op->iter.set;
if (op->encoding == REDIS_ENCODING_INTSET) {
if (!intsetGet(it->is.is,it->is.ii,&val->ell))
return 0;
val->score = 1.0;
/* Move to next element. */
it->is.ii++;
} else if (op->encoding == REDIS_ENCODING_HT) {
if (it->ht.de == NULL)
return 0;
val->ele = dictGetEntryKey(it->ht.de);
val->score = 1.0;
/* Move to next element. */
it->ht.de = dictNext(it->ht.di);
} else {
redisPanic("Unknown set encoding");
}
} else if (op->type == REDIS_ZSET) {
iterzset *it = &op->iter.zset;
if (op->encoding == REDIS_ENCODING_ZIPLIST) {
/* No need to check both, but better be explicit. */
if (it->zl.eptr == NULL || it->zl.sptr == NULL)
return 0;
redisAssert(ziplistGet(it->zl.eptr,&val->estr,&val->elen,&val->ell));
val->score = zzlGetScore(it->zl.sptr);
/* Move to next element. */
zzlNext(it->zl.zl,&it->zl.eptr,&it->zl.sptr);
} else if (op->encoding == REDIS_ENCODING_RAW) {
if (it->sl.node == NULL)
return 0;
val->ele = it->sl.node->obj;
val->score = it->sl.node->score;
/* Move to next element. */
it->sl.node = it->sl.node->level[0].forward;
} else {
redisPanic("Unknown sorted set encoding");
}
} else {
redisPanic("Unsupported type");
}
return 1;
}
int zuiLongLongFromValue(zsetopval *val) {
if (!(val->flags & OPVAL_DIRTY_LL)) {
val->flags |= OPVAL_DIRTY_LL;
if (val->ele != NULL) {
if (val->ele->encoding == REDIS_ENCODING_INT) {
val->ell = (long)val->ele->ptr;
val->flags |= OPVAL_VALID_LL;
} else if (val->ele->encoding == REDIS_ENCODING_RAW) {
if (string2ll(val->ele->ptr,sdslen(val->ele->ptr),&val->ell))
val->flags |= OPVAL_VALID_LL;
} else {
redisPanic("Unsupported element encoding");
}
} else if (val->estr != NULL) {
if (string2ll((char*)val->estr,val->elen,&val->ell))
val->flags |= OPVAL_VALID_LL;
} else {
/* The long long was already set, flag as valid. */
val->flags |= OPVAL_VALID_LL;
}
}
return val->flags & OPVAL_VALID_LL;
}
robj *zuiObjectFromValue(zsetopval *val) {
if (val->ele == NULL) {
if (val->estr != NULL) {
val->ele = createStringObject((char*)val->estr,val->elen);
} else {
val->ele = createStringObjectFromLongLong(val->ell);
}
val->flags |= OPVAL_DIRTY_ROBJ;
}
return val->ele;
}
int zuiBufferFromValue(zsetopval *val) {
if (val->estr == NULL) {
if (val->ele != NULL) {
if (val->ele->encoding == REDIS_ENCODING_INT) {
val->elen = ll2string((char*)val->_buf,sizeof(val->_buf),(long)val->ele->ptr);
val->estr = val->_buf;
} else if (val->ele->encoding == REDIS_ENCODING_RAW) {
val->elen = sdslen(val->ele->ptr);
val->estr = val->ele->ptr;
} else {
redisPanic("Unsupported element encoding");
}
} else {
val->elen = ll2string((char*)val->_buf,sizeof(val->_buf),val->ell);
val->estr = val->_buf;
}
}
return 1;
}
/* Find value pointed to by val in the source pointer to by op. When found,
* return 1 and store its score in target. Return 0 otherwise. */
int zuiFind(zsetopsrc *op, zsetopval *val, double *score) {
if (op->subject == NULL)
return 0;
if (op->type == REDIS_SET) {
iterset *it = &op->iter.set;
if (op->encoding == REDIS_ENCODING_INTSET) {
if (zuiLongLongFromValue(val) && intsetFind(it->is.is,val->ell)) {
*score = 1.0;
return 1;
} else {
return 0;
}
} else if (op->encoding == REDIS_ENCODING_HT) {
zuiObjectFromValue(val);
if (dictFind(it->ht.dict,val->ele) != NULL) {
*score = 1.0;
return 1;
} else {
return 0;
}
} else {
redisPanic("Unknown set encoding");
}
} else if (op->type == REDIS_ZSET) {
iterzset *it = &op->iter.zset;
zuiObjectFromValue(val);
if (op->encoding == REDIS_ENCODING_ZIPLIST) {
if (zzlFind(op->subject,val->ele,score) != NULL) {
/* Score is already set by zzlFind. */
return 1;
} else {
return 0;
}
} else if (op->encoding == REDIS_ENCODING_RAW) {
dictEntry *de;
if ((de = dictFind(it->sl.zs->dict,val->ele)) != NULL) {
*score = *(double*)dictGetEntryVal(de);
return 1;
} else {
return 0;
}
} else {
redisPanic("Unknown sorted set encoding");
}
} else {
redisPanic("Unsupported type");
}
}
int zuiCompareByCardinality(const void *s1, const void *s2) {
return zuiLength((zsetopsrc*)s1) - zuiLength((zsetopsrc*)s2);
}
}
#define REDIS_AGGR_SUM 1
#define REDIS_AGGR_SUM 1
...
@@ -1113,11 +1430,11 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
...
@@ -1113,11 +1430,11 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
int i, j, setnum;
int i, j, setnum;
int aggregate = REDIS_AGGR_SUM;
int aggregate = REDIS_AGGR_SUM;
zsetopsrc *src;
zsetopsrc *src;
zsetopval zval;
robj *tmp;
robj *dstobj;
robj *dstobj;
zset *dstzset;
zset *dstzset;
zskiplistNode *znode;
zskiplistNode *znode;
dictIterator *di;
dictEntry *de;
int touched = 0;
int touched = 0;
/* expect setnum input keys to be given */
/* expect setnum input keys to be given */
...
@@ -1135,24 +1452,24 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
...
@@ -1135,24 +1452,24 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
}
}
/* read keys to be used for input */
/* read keys to be used for input */
src = z
m
alloc(sizeof(zsetopsrc) * setnum);
src = z
c
alloc(sizeof(zsetopsrc) * setnum);
for (i = 0, j = 3; i < setnum; i++, j++) {
for (i = 0, j = 3; i < setnum; i++, j++) {
robj *obj = lookupKeyWrite(c->db,c->argv[j]);
robj *obj = lookupKeyWrite(c->db,c->argv[j]);
if (!obj) {
if (obj != NULL) {
src[i].dict = NULL;
if (obj->type != REDIS_ZSET && obj->type != REDIS_SET) {
} else {
if (obj->type == REDIS_ZSET) {
src[i].dict = ((zset*)obj->ptr)->dict;
} else if (obj->type == REDIS_SET) {
src[i].dict = (obj->ptr);
} else {
zfree(src);
zfree(src);
addReply(c,shared.wrongtypeerr);
addReply(c,shared.wrongtypeerr);
return;
return;
}
}
src[i].subject = obj;
src[i].type = obj->type;
src[i].encoding = obj->encoding;
} else {
src[i].subject = NULL;
}
}
/*
d
efault all weights to 1 */
/*
D
efault all weights to 1
.
*/
src[i].weight = 1.0;
src[i].weight = 1.0;
}
}
...
@@ -1193,82 +1510,82 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
...
@@ -1193,82 +1510,82 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
}
}
}
}
for (i = 0; i < setnum; i++)
zuiInitIterator(&src[i]);
/* sort sets from the smallest to largest, this will improve our
/* sort sets from the smallest to largest, this will improve our
* algorithm's performance */
* algorithm's performance */
qsort(src,setnum,sizeof(zsetopsrc),
qsort
Compare
Zsetopsrc
ByCardinality);
qsort(src,setnum,sizeof(zsetopsrc),
zui
CompareByCardinality);
dstobj = createZsetObject();
dstobj = createZsetObject();
dstzset = dstobj->ptr;
dstzset = dstobj->ptr;
if (op == REDIS_OP_INTER) {
if (op == REDIS_OP_INTER) {
/* skip going over all entries if the smallest zset is NULL or empty */
/* Skip everything if the smallest input is empty. */
if (src[0].dict && dictSize(src[0].dict) > 0) {
if (zuiLength(&src[0]) > 0) {
/* precondition: as src[0].dict is non-empty and the zsets are ordered
/* Precondition: as src[0] is non-empty and the inputs are ordered
* from small to large, all src[i > 0].dict are non-empty too */
* by size, all src[i > 0] are non-empty too. */
di = dictGetIterator(src[0].dict);
while (zuiNext(&src[0],&zval)) {
while((de = dictNext(di)) != NULL) {
double score, value;
double score, value;
score = src[0].weight * z
unionInterDictValue(de)
;
score = src[0].weight * z
val.score
;
for (j = 1; j < setnum; j++) {
for (j = 1; j < setnum; j++) {
dictEntry *other = dictFind(src[j].dict,dictGetEntryKey(de));
if (zuiFind(&src[j],&zval,&value)) {
if (other) {
value *= src[j].weight;
value = src[j].weight * zunionInterDictValue(other);
zunionInterAggregate(&score,value,aggregate);
zunionInterAggregate(&score,value,aggregate);
} else {
} else {
break;
break;
}
}
}
}
/* Only continue when present in every
source dic
t. */
/* Only continue when present in every
inpu
t. */
if (j == setnum) {
if (j == setnum) {
robj *o = dictGetEntryKey(de
);
tmp = zuiObjectFromValue(&zval
);
znode = zslInsert(dstzset->zsl,score,
o
);
znode = zslInsert(dstzset->zsl,score,
tmp
);
incrRefCount(
o
); /* added to skiplist */
incrRefCount(
tmp
); /* added to skiplist */
dictAdd(dstzset->dict,
o
,&znode->score);
dictAdd(dstzset->dict,
tmp
,&znode->score);
incrRefCount(
o
); /* added to dictionary */
incrRefCount(
tmp
); /* added to dictionary */
}
}
}
}
dictReleaseIterator(di);
}
}
} else if (op == REDIS_OP_UNION) {
} else if (op == REDIS_OP_UNION) {
for (i = 0; i < setnum; i++) {
for (i = 0; i < setnum; i++) {
if (!src[i].dict) continue;
if (zuiLength(&src[0]) == 0)
continue;
di = dictGetIterator(src[i].dict);
while (zuiNext(&src[i],&zval)) {
while((de = dictNext(di)) != NULL) {
double score, value;
double score, value;
/*
s
kip key when already processed */
/*
S
kip key when already processed */
if (dictFind(dstzset->dict,
dictGetEntryKey(de
)) != NULL)
if (dictFind(dstzset->dict,
zuiObjectFromValue(&zval
)) != NULL)
continue;
continue;
/*
i
nitialize score */
/*
I
nitialize score */
score = src[i].weight * z
unionInterDictValue(de)
;
score = src[i].weight * z
val.score
;
/*
b
ecause the
zse
ts are sorted by size, its only possible
/*
B
ecause the
inpu
ts are sorted by size, it
'
s only possible
* for sets at larger indices to hold this e
ntry
*/
* for sets at larger indices to hold this e
lement.
*/
for (j = (i+1); j < setnum; j++) {
for (j = (i+1); j < setnum; j++) {
dictEntry *other = dictFind(src[j].dict,dictGetEntryKey(de));
if (zuiFind(&src[j],&zval,&value)) {
if (other) {
value *= src[j].weight;
value = src[j].weight * zunionInterDictValue(other);
zunionInterAggregate(&score,value,aggregate);
zunionInterAggregate(&score,value,aggregate);
}
}
}
}
robj *o = dictGetEntryKey(de
);
tmp = zuiObjectFromValue(&zval
);
znode = zslInsert(dstzset->zsl,score,
o
);
znode = zslInsert(dstzset->zsl,score,
tmp
);
incrRefCount(
o
); /* added to skiplist */
incrRefCount(
zval.ele
); /* added to skiplist */
dictAdd(dstzset->dict,
o
,&znode->score);
dictAdd(dstzset->dict,
tmp
,&znode->score);
incrRefCount(
o
); /* added to dictionary */
incrRefCount(
zval.ele
); /* added to dictionary */
}
}
dictReleaseIterator(di);
}
}
} else {
} else {
/* unknown operator */
redisPanic("Unknown operator");
redisAssert(op == REDIS_OP_INTER || op == REDIS_OP_UNION);
}
}
for (i = 0; i < setnum; i++)
zuiClearIterator(&src[i]);
if (dbDelete(c->db,dstkey)) {
if (dbDelete(c->db,dstkey)) {
signalModifiedKey(c->db,dstkey);
signalModifiedKey(c->db,dstkey);
touched = 1;
touched = 1;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment