Unverified Commit 4e472a1a authored by Viktor Söderqvist's avatar Viktor Söderqvist Committed by GitHub
Browse files

Listpack encoding for sets (#11290)

Small sets with not only integer elements are listpack encoded, by default
up to 128 elements, max 64 bytes per element, new config `set-max-listpack-entries`
and `set-max-listpack-value`. This saves memory for small sets compared to using a hashtable.

Sets with only integers, even very small sets, are still intset encoded (up to 1G
limit, etc.). Larger sets are hashtable encoded.

This PR increments the RDB version, and has an effect on OBJECT ENCODING

Possible conversions when elements are added:

    intset -> listpack
    listpack -> hashtable
    intset -> hashtable

Note: No conversion happens when elements are deleted. If all elements are
deleted and then added again, the set is deleted and recreated, thus implicitly
converted to a smaller encoding.
parent 07d18706
...@@ -1951,13 +1951,20 @@ list-max-listpack-size -2 ...@@ -1951,13 +1951,20 @@ list-max-listpack-size -2
# etc. # etc.
list-compress-depth 0 list-compress-depth 0
# Sets have a special encoding in just one case: when a set is composed # Sets have a special encoding when a set is composed
# of just strings that happen to be integers in radix 10 in the range # of just strings that happen to be integers in radix 10 in the range
# of 64 bit signed integers. # of 64 bit signed integers.
# The following configuration setting sets the limit in the size of the # The following configuration setting sets the limit in the size of the
# set in order to use this special memory saving encoding. # set in order to use this special memory saving encoding.
set-max-intset-entries 512 set-max-intset-entries 512
# Sets containing non-integer values are also encoded using a memory efficient
# data structure when they have a small number of entries, and the biggest entry
# does not exceed a given threshold. These thresholds can be configured using
# the following directives.
set-max-listpack-entries 128
set-max-listpack-value 64
# Similarly to hashes and lists, sorted sets are also specially encoded in # Similarly to hashes and lists, sorted sets are also specially encoded in
# order to save a lot of space. This encoding is only used when the length and # order to save a lot of space. This encoding is only used when the length and
# elements of a sorted set are below the following limits: # elements of a sorted set are below the following limits:
......
...@@ -1818,56 +1818,31 @@ int rewriteListObject(rio *r, robj *key, robj *o) { ...@@ -1818,56 +1818,31 @@ int rewriteListObject(rio *r, robj *key, robj *o) {
* The function returns 0 on error, 1 on success. */ * The function returns 0 on error, 1 on success. */
int rewriteSetObject(rio *r, robj *key, robj *o) { int rewriteSetObject(rio *r, robj *key, robj *o) {
long long count = 0, items = setTypeSize(o); long long count = 0, items = setTypeSize(o);
setTypeIterator *si = setTypeInitIterator(o);
if (o->encoding == OBJ_ENCODING_INTSET) { char *str;
int ii = 0; size_t len;
int64_t llval; int64_t llval;
while (setTypeNext(si, &str, &len, &llval) != -1) {
while(intsetGet(o->ptr,ii++,&llval)) { if (count == 0) {
if (count == 0) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items;
AOF_REWRITE_ITEMS_PER_CMD : items; if (!rioWriteBulkCount(r,'*',2+cmd_items) ||
!rioWriteBulkString(r,"SADD",4) ||
if (!rioWriteBulkCount(r,'*',2+cmd_items) || !rioWriteBulkObject(r,key))
!rioWriteBulkString(r,"SADD",4) || {
!rioWriteBulkObject(r,key)) return 0;
{
return 0;
}
} }
if (!rioWriteBulkLongLong(r,llval)) return 0;
if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
items--;
} }
} else if (o->encoding == OBJ_ENCODING_HT) { size_t written = str ?
dictIterator *di = dictGetIterator(o->ptr); rioWriteBulkString(r, str, len) : rioWriteBulkLongLong(r, llval);
dictEntry *de; if (!written) {
setTypeReleaseIterator(si);
while((de = dictNext(di)) != NULL) { return 0;
sds ele = dictGetKey(de);
if (count == 0) {
int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
AOF_REWRITE_ITEMS_PER_CMD : items;
if (!rioWriteBulkCount(r,'*',2+cmd_items) ||
!rioWriteBulkString(r,"SADD",4) ||
!rioWriteBulkObject(r,key))
{
dictReleaseIterator(di);
return 0;
}
}
if (!rioWriteBulkString(r,ele,sdslen(ele))) {
dictReleaseIterator(di);
return 0;
}
if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
items--;
} }
dictReleaseIterator(di); if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
} else { items--;
serverPanic("Unknown set encoding");
} }
setTypeReleaseIterator(si);
return 1; return 1;
} }
......
...@@ -3130,6 +3130,8 @@ standardConfig static_configs[] = { ...@@ -3130,6 +3130,8 @@ standardConfig static_configs[] = {
/* Size_t configs */ /* Size_t configs */
createSizeTConfig("hash-max-listpack-entries", "hash-max-ziplist-entries", MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_listpack_entries, 512, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("hash-max-listpack-entries", "hash-max-ziplist-entries", MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_listpack_entries, 512, INTEGER_CONFIG, NULL, NULL),
createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL),
createSizeTConfig("set-max-listpack-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_listpack_entries, 128, INTEGER_CONFIG, NULL, NULL),
createSizeTConfig("set-max-listpack-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_listpack_value, 64, INTEGER_CONFIG, NULL, NULL),
createSizeTConfig("zset-max-listpack-entries", "zset-max-ziplist-entries", MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_listpack_entries, 128, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("zset-max-listpack-entries", "zset-max-ziplist-entries", MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_listpack_entries, 128, INTEGER_CONFIG, NULL, NULL),
createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */
createSizeTConfig("hash-max-listpack-value", "hash-max-ziplist-value", MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_listpack_value, 64, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("hash-max-listpack-value", "hash-max-ziplist-value", MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_listpack_value, 64, MEMORY_CONFIG, NULL, NULL),
......
...@@ -915,14 +915,16 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { ...@@ -915,14 +915,16 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) {
} while (cursor && } while (cursor &&
maxiterations-- && maxiterations-- &&
listLength(keys) < (unsigned long)count); listLength(keys) < (unsigned long)count);
} else if (o->type == OBJ_SET) { } else if (o->type == OBJ_SET && o->encoding == OBJ_ENCODING_INTSET) {
int pos = 0; int pos = 0;
int64_t ll; int64_t ll;
while(intsetGet(o->ptr,pos++,&ll)) while(intsetGet(o->ptr,pos++,&ll))
listAddNodeTail(keys,createStringObjectFromLongLong(ll)); listAddNodeTail(keys,createStringObjectFromLongLong(ll));
cursor = 0; cursor = 0;
} else if (o->type == OBJ_HASH || o->type == OBJ_ZSET) { } else if ((o->type == OBJ_HASH || o->type == OBJ_ZSET || o->type == OBJ_SET) &&
o->encoding == OBJ_ENCODING_LISTPACK)
{
unsigned char *p = lpFirst(o->ptr); unsigned char *p = lpFirst(o->ptr);
unsigned char *vstr; unsigned char *vstr;
int64_t vlen; int64_t vlen;
......
...@@ -874,10 +874,12 @@ long defragKey(redisDb *db, dictEntry *de) { ...@@ -874,10 +874,12 @@ long defragKey(redisDb *db, dictEntry *de) {
} else if (ob->type == OBJ_SET) { } else if (ob->type == OBJ_SET) {
if (ob->encoding == OBJ_ENCODING_HT) { if (ob->encoding == OBJ_ENCODING_HT) {
defragged += defragSet(db, de); defragged += defragSet(db, de);
} else if (ob->encoding == OBJ_ENCODING_INTSET) { } else if (ob->encoding == OBJ_ENCODING_INTSET ||
intset *newis, *is = ob->ptr; ob->encoding == OBJ_ENCODING_LISTPACK)
if ((newis = activeDefragAlloc(is))) {
defragged++, ob->ptr = newis; void *newptr, *ptr = ob->ptr;
if ((newptr = activeDefragAlloc(ptr)))
defragged++, ob->ptr = newptr;
} else { } else {
serverPanic("Unknown set encoding"); serverPanic("Unknown set encoding");
} }
......
...@@ -265,6 +265,17 @@ int64_t intsetRandom(intset *is) { ...@@ -265,6 +265,17 @@ int64_t intsetRandom(intset *is) {
return _intsetGet(is,rand()%len); return _intsetGet(is,rand()%len);
} }
/* Return the largest member. */
int64_t intsetMax(intset *is) {
uint32_t len = intrev32ifbe(is->length);
return _intsetGet(is, len - 1);
}
/* Return the smallest member. */
int64_t intsetMin(intset *is) {
return _intsetGet(is, 0);
}
/* Get the value at the given position. When this position is /* Get the value at the given position. When this position is
* out of range the function returns 0, when in range it returns 1. */ * out of range the function returns 0, when in range it returns 1. */
uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value) { uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value) {
...@@ -425,6 +436,8 @@ int intsetTest(int argc, char **argv, int flags) { ...@@ -425,6 +436,8 @@ int intsetTest(int argc, char **argv, int flags) {
is = intsetAdd(is,6,&success); assert(success); is = intsetAdd(is,6,&success); assert(success);
is = intsetAdd(is,4,&success); assert(success); is = intsetAdd(is,4,&success); assert(success);
is = intsetAdd(is,4,&success); assert(!success); is = intsetAdd(is,4,&success); assert(!success);
assert(6 == intsetMax(is));
assert(4 == intsetMin(is));
ok(); ok();
zfree(is); zfree(is);
} }
......
...@@ -43,6 +43,8 @@ intset *intsetAdd(intset *is, int64_t value, uint8_t *success); ...@@ -43,6 +43,8 @@ intset *intsetAdd(intset *is, int64_t value, uint8_t *success);
intset *intsetRemove(intset *is, int64_t value, int *success); intset *intsetRemove(intset *is, int64_t value, int *success);
uint8_t intsetFind(intset *is, int64_t value); uint8_t intsetFind(intset *is, int64_t value);
int64_t intsetRandom(intset *is); int64_t intsetRandom(intset *is);
int64_t intsetMax(intset *is);
int64_t intsetMin(intset *is);
uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value); uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value);
uint32_t intsetLen(const intset *is); uint32_t intsetLen(const intset *is);
size_t intsetBlobLen(intset *is); size_t intsetBlobLen(intset *is);
......
...@@ -1063,6 +1063,55 @@ unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num) { ...@@ -1063,6 +1063,55 @@ unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num) {
return lp; return lp;
} }
/* Delete the elements 'ps' passed as an array of 'count' element pointers and
* return the resulting listpack. The elements must be given in the same order
* as they apper in the listpack. */
unsigned char *lpBatchDelete(unsigned char *lp, unsigned char **ps, unsigned long count) {
if (count == 0) return lp;
unsigned char *dst = ps[0];
size_t total_bytes = lpGetTotalBytes(lp);
unsigned char *lp_end = lp + total_bytes; /* After the EOF element. */
assert(lp_end[-1] == LP_EOF);
/*
* ----+--------+-----------+--------+---------+-----+---+
* ... | Delete | Keep | Delete | Keep | ... |EOF|
* ... |xxxxxxxx| |xxxxxxxx| | ... | |
* ----+--------+-----------+--------+---------+-----+---+
* ^ ^ ^ ^
* | | | |
* ps[i] | ps[i+1] |
* skip keep_start keep_end lp_end
*
* The loop memmoves the bytes between keep_start and keep_end to dst.
*/
for (unsigned long i = 0; i < count; i++) {
unsigned char *skip = ps[i];
assert(skip != NULL && skip[0] != LP_EOF);
unsigned char *keep_start = lpSkip(skip);
unsigned char *keep_end;
if (i + 1 < count) {
keep_end = ps[i + 1];
/* Deleting consecutive elements. Nothing to keep between them. */
if (keep_start == keep_end) continue;
} else {
/* Keep the rest of the listpack including the EOF marker. */
keep_end = lp_end;
}
assert(keep_end > keep_start);
size_t bytes_to_keep = keep_end - keep_start;
memmove(dst, keep_start, bytes_to_keep);
dst += bytes_to_keep;
}
/* Update total size and num elements. */
size_t deleted_bytes = lp_end - dst;
total_bytes -= deleted_bytes;
assert(lp[total_bytes - 1] == LP_EOF);
lpSetTotalBytes(lp, total_bytes);
uint32_t numele = lpGetNumElements(lp);
if (numele != LP_HDR_NUMELE_UNKNOWN) lpSetNumElements(lp, numele - count);
return lpShrinkToFit(lp);
}
/* Merge listpacks 'first' and 'second' by appending 'second' to 'first'. /* Merge listpacks 'first' and 'second' by appending 'second' to 'first'.
* *
* NOTE: The larger listpack is reallocated to contain the new merged listpack. * NOTE: The larger listpack is reallocated to contain the new merged listpack.
...@@ -1383,6 +1432,43 @@ void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *k ...@@ -1383,6 +1432,43 @@ void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *k
val->sval = lpGetValue(p, &(val->slen), &(val->lval)); val->sval = lpGetValue(p, &(val->slen), &(val->lval));
} }
/* Randomly select 'count' entries and store them in the 'entries' array, which
* needs to have space for 'count' listpackEntry structs. The order is random
* and duplicates are possible. */
void lpRandomEntries(unsigned char *lp, unsigned int count, listpackEntry *entries) {
struct pick {
unsigned int index;
unsigned int order;
} *picks = lp_malloc(count * sizeof(struct pick));
unsigned int total_size = lpLength(lp);
assert(total_size);
for (unsigned int i = 0; i < count; i++) {
picks[i].index = rand() % total_size;
picks[i].order = i;
}
/* Sort by index. */
qsort(picks, count, sizeof(struct pick), uintCompare);
/* Iterate over listpack in index order and store the values in the entries
* array respecting the original order. */
unsigned char *p = lpFirst(lp);
unsigned int j = 0; /* index in listpack */
for (unsigned int i = 0; i < count; i++) {
/* Advance listpack pointer to until we reach 'index' listpack. */
while (j < picks[i].index) {
p = lpNext(lp, p);
j++;
}
int storeorder = picks[i].order;
unsigned int len = 0;
long long llval = 0;
unsigned char *str = lpGetValue(p, &len, &llval);
lpSaveValue(str, len, llval, &entries[storeorder]);
}
lp_free(picks);
}
/* Randomly select count of key value pairs and store into 'keys' and /* Randomly select count of key value pairs and store into 'keys' and
* 'vals' args. The order of the picked entries is random, and the selections * 'vals' args. The order of the picked entries is random, and the selections
* are non-unique (repetitions are possible). * are non-unique (repetitions are possible).
...@@ -1449,34 +1535,83 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack ...@@ -1449,34 +1535,83 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack
if (count > total_size) if (count > total_size)
count = total_size; count = total_size;
/* To only iterate once, every time we try to pick a member, the probability
* we pick it is the quotient of the count left we want to pick and the
* count still we haven't visited in the dict, this way, we could make every
* member be equally picked.*/
p = lpFirst(lp); p = lpFirst(lp);
unsigned int picked = 0, remaining = count; unsigned int picked = 0, remaining = count;
while (picked < count && p) { while (picked < count && p) {
double randomDouble = ((double)rand()) / RAND_MAX; assert((p = lpNextRandom(lp, p, &index, remaining, 1)));
double threshold = ((double)remaining) / (total_size - index); key = lpGetValue(p, &klen, &klval);
if (randomDouble <= threshold) { lpSaveValue(key, klen, klval, &keys[picked]);
assert((p = lpNext(lp, p)));
index++;
if (vals) {
key = lpGetValue(p, &klen, &klval); key = lpGetValue(p, &klen, &klval);
lpSaveValue(key, klen, klval, &keys[picked]); lpSaveValue(key, klen, klval, &vals[picked]);
assert((p = lpNext(lp, p)));
if (vals) {
key = lpGetValue(p, &klen, &klval);
lpSaveValue(key, klen, klval, &vals[picked]);
}
remaining--;
picked++;
} else {
assert((p = lpNext(lp, p)));
} }
p = lpNext(lp, p); p = lpNext(lp, p);
remaining--;
picked++;
index++; index++;
} }
return picked; return picked;
} }
/* Iterates forward to the "next random" element, given we are yet to pick
* 'remaining' unique elements between the starting element 'p' (inclusive) and
* the end of the list. The 'index' needs to be initialized according to the
* current zero-based index matching the position of the starting element 'p'
* and is updated to match the returned element's zero-based index. If
* 'even_only' is nonzero, an element with an even index is picked, which is
* useful if the listpack represents a key-value pair sequence.
*
* Note that this function can return p. In order to skip the previously
* returned element, you need to call lpNext() or lpDelete() after each call to
* lpNextRandom(). Idea:
*
* assert(remaining <= lpLength(lp));
* p = lpFirst(lp);
* i = 0;
* while (remaining > 0) {
* p = lpNextRandom(lp, p, &i, remaining--, 0);
*
* // ... Do stuff with p ...
*
* p = lpNext(lp, p);
* i++;
* }
*/
unsigned char *lpNextRandom(unsigned char *lp, unsigned char *p, unsigned int *index,
unsigned int remaining, int even_only)
{
/* To only iterate once, every time we try to pick a member, the probability
* we pick it is the quotient of the count left we want to pick and the
* count still we haven't visited. This way, we could make every member be
* equally likely to be picked. */
unsigned int i = *index;
unsigned int total_size = lpLength(lp);
while (i < total_size && p != NULL) {
if (even_only && i % 2 != 0) {
p = lpNext(lp, p);
i++;
continue;
}
/* Do we pick this element? */
unsigned int available = total_size - i;
if (even_only) available /= 2;
double randomDouble = ((double)rand()) / RAND_MAX;
double threshold = ((double)remaining) / available;
if (randomDouble <= threshold) {
*index = i;
return p;
}
p = lpNext(lp, p);
i++;
}
return NULL;
}
/* Print info of listpack which is used in debugCommand */ /* Print info of listpack which is used in debugCommand */
void lpRepr(unsigned char *lp) { void lpRepr(unsigned char *lp) {
unsigned char *p, *vstr; unsigned char *p, *vstr;
...@@ -1902,6 +2037,21 @@ int listpackTest(int argc, char *argv[], int flags) { ...@@ -1902,6 +2037,21 @@ int listpackTest(int argc, char *argv[], int flags) {
zfree(lp); zfree(lp);
} }
TEST("Batch delete") {
unsigned char *lp = createList(); /* char *mixlist[] = {"hello", "foo", "quux", "1024"} */
assert(lpLength(lp) == 4); /* Pre-condition */
unsigned char *p0 = lpFirst(lp),
*p1 = lpNext(lp, p0),
*p2 = lpNext(lp, p1),
*p3 = lpNext(lp, p2);
unsigned char *ps[] = {p0, p1, p3};
lp = lpBatchDelete(lp, ps, 3);
assert(lpLength(lp) == 1);
verifyEntry(lpFirst(lp), (unsigned char*)mixlist[2], strlen(mixlist[2]));
assert(lpValidateIntegrity(lp, lpBytes(lp), 1, NULL, NULL) == 1);
lpFree(lp);
}
TEST("Delete foo while iterating") { TEST("Delete foo while iterating") {
lp = createList(); lp = createList();
p = lpFirst(lp); p = lpFirst(lp);
...@@ -2048,6 +2198,82 @@ int listpackTest(int argc, char *argv[], int flags) { ...@@ -2048,6 +2198,82 @@ int listpackTest(int argc, char *argv[], int flags) {
zfree(lp3); zfree(lp3);
} }
TEST("lpNextRandom normal usage") {
/* Create some data */
unsigned char *lp = lpNew(0);
unsigned char buf[100] = "asdf";
unsigned int size = 100;
for (size_t i = 0; i < size; i++) {
lp = lpAppend(lp, buf, i);
}
assert(lpLength(lp) == size);
/* Pick a subset of the elements of every possible subset size */
for (unsigned int count = 0; count <= size; count++) {
unsigned int remaining = count;
unsigned char *p = lpFirst(lp);
unsigned char *prev = NULL;
unsigned index = 0;
while (remaining > 0) {
assert(p != NULL);
p = lpNextRandom(lp, p, &index, remaining--, 0);
assert(p != NULL);
assert(p != prev);
prev = p;
p = lpNext(lp, p);
index++;
}
}
}
TEST("lpNextRandom corner cases") {
unsigned char *lp = lpNew(0);
unsigned i = 0;
/* Pick from empty listpack returns NULL. */
assert(lpNextRandom(lp, NULL, &i, 2, 0) == NULL);
/* Add some elements and find their pointers within the listpack. */
lp = lpAppend(lp, (unsigned char *)"abc", 3);
lp = lpAppend(lp, (unsigned char *)"def", 3);
lp = lpAppend(lp, (unsigned char *)"ghi", 3);
assert(lpLength(lp) == 3);
unsigned char *p0 = lpFirst(lp);
unsigned char *p1 = lpNext(lp, p0);
unsigned char *p2 = lpNext(lp, p1);
assert(lpNext(lp, p2) == NULL);
/* Pick zero elements returns NULL. */
i = 0; assert(lpNextRandom(lp, lpFirst(lp), &i, 0, 0) == NULL);
/* Pick all returns all. */
i = 0; assert(lpNextRandom(lp, p0, &i, 3, 0) == p0 && i == 0);
i = 1; assert(lpNextRandom(lp, p1, &i, 2, 0) == p1 && i == 1);
i = 2; assert(lpNextRandom(lp, p2, &i, 1, 0) == p2 && i == 2);
/* Pick more than one when there's only one left returns the last one. */
i = 2; assert(lpNextRandom(lp, p2, &i, 42, 0) == p2 && i == 2);
/* Pick all even elements returns p0 and p2. */
i = 0; assert(lpNextRandom(lp, p0, &i, 10, 1) == p0 && i == 0);
i = 1; assert(lpNextRandom(lp, p1, &i, 10, 1) == p2 && i == 2);
/* Don't crash even for bad index. */
for (int j = 0; j < 100; j++) {
unsigned char *p;
switch (j % 4) {
case 0: p = p0; break;
case 1: p = p1; break;
case 2: p = p2; break;
case 3: p = NULL; break;
}
i = j % 7;
unsigned int remaining = j % 5;
p = lpNextRandom(lp, p, &i, remaining, 0);
assert(p == p0 || p == p1 || p == p2 || p == NULL);
}
}
TEST("Random pair with one element") { TEST("Random pair with one element") {
listpackEntry key, val; listpackEntry key, val;
unsigned char *lp = lpNew(0); unsigned char *lp = lpNew(0);
......
...@@ -70,6 +70,7 @@ unsigned char *lpReplaceInteger(unsigned char *lp, unsigned char **p, long long ...@@ -70,6 +70,7 @@ unsigned char *lpReplaceInteger(unsigned char *lp, unsigned char **p, long long
unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **newp); unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **newp);
unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsigned long num); unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsigned long num);
unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num); unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num);
unsigned char *lpBatchDelete(unsigned char *lp, unsigned char **ps, unsigned long count);
unsigned char *lpMerge(unsigned char **first, unsigned char **second); unsigned char *lpMerge(unsigned char **first, unsigned char **second);
unsigned long lpLength(unsigned char *lp); unsigned long lpLength(unsigned char *lp);
unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf); unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf);
...@@ -90,6 +91,9 @@ unsigned int lpCompare(unsigned char *p, unsigned char *s, uint32_t slen); ...@@ -90,6 +91,9 @@ unsigned int lpCompare(unsigned char *p, unsigned char *s, uint32_t slen);
void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *key, listpackEntry *val); void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *key, listpackEntry *val);
void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals); void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals);
unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals); unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals);
void lpRandomEntries(unsigned char *lp, unsigned int count, listpackEntry *entries);
unsigned char *lpNextRandom(unsigned char *lp, unsigned char *p, unsigned int *index,
unsigned int remaining, int even_only);
int lpSafeToAdd(unsigned char* lp, size_t add); int lpSafeToAdd(unsigned char* lp, size_t add);
void lpRepr(unsigned char *lp); void lpRepr(unsigned char *lp);
......
...@@ -247,6 +247,13 @@ robj *createIntsetObject(void) { ...@@ -247,6 +247,13 @@ robj *createIntsetObject(void) {
return o; return o;
} }
robj *createSetListpackObject(void) {
unsigned char *lp = lpNew(0);
robj *o = createObject(OBJ_SET, lp);
o->encoding = OBJ_ENCODING_LISTPACK;
return o;
}
robj *createHashObject(void) { robj *createHashObject(void) {
unsigned char *zl = lpNew(0); unsigned char *zl = lpNew(0);
robj *o = createObject(OBJ_HASH, zl); robj *o = createObject(OBJ_HASH, zl);
...@@ -306,6 +313,7 @@ void freeSetObject(robj *o) { ...@@ -306,6 +313,7 @@ void freeSetObject(robj *o) {
dictRelease((dict*) o->ptr); dictRelease((dict*) o->ptr);
break; break;
case OBJ_ENCODING_INTSET: case OBJ_ENCODING_INTSET:
case OBJ_ENCODING_LISTPACK:
zfree(o->ptr); zfree(o->ptr);
break; break;
default: default:
...@@ -441,6 +449,8 @@ void dismissSetObject(robj *o, size_t size_hint) { ...@@ -441,6 +449,8 @@ void dismissSetObject(robj *o, size_t size_hint) {
dismissMemory(set->ht_table[1], DICTHT_SIZE(set->ht_size_exp[1])*sizeof(dictEntry*)); dismissMemory(set->ht_table[1], DICTHT_SIZE(set->ht_size_exp[1])*sizeof(dictEntry*));
} else if (o->encoding == OBJ_ENCODING_INTSET) { } else if (o->encoding == OBJ_ENCODING_INTSET) {
dismissMemory(o->ptr, intsetBlobLen((intset*)o->ptr)); dismissMemory(o->ptr, intsetBlobLen((intset*)o->ptr));
} else if (o->encoding == OBJ_ENCODING_LISTPACK) {
dismissMemory(o->ptr, lpBytes((unsigned char *)o->ptr));
} else { } else {
serverPanic("Unknown set encoding type"); serverPanic("Unknown set encoding type");
} }
......
...@@ -665,6 +665,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) { ...@@ -665,6 +665,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) {
return rdbSaveType(rdb,RDB_TYPE_SET_INTSET); return rdbSaveType(rdb,RDB_TYPE_SET_INTSET);
else if (o->encoding == OBJ_ENCODING_HT) else if (o->encoding == OBJ_ENCODING_HT)
return rdbSaveType(rdb,RDB_TYPE_SET); return rdbSaveType(rdb,RDB_TYPE_SET);
else if (o->encoding == OBJ_ENCODING_LISTPACK)
return rdbSaveType(rdb,RDB_TYPE_SET_LISTPACK);
else else
serverPanic("Unknown set encoding"); serverPanic("Unknown set encoding");
case OBJ_ZSET: case OBJ_ZSET:
...@@ -858,6 +860,10 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { ...@@ -858,6 +860,10 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) {
if ((n = rdbSaveRawString(rdb,o->ptr,l)) == -1) return -1; if ((n = rdbSaveRawString(rdb,o->ptr,l)) == -1) return -1;
nwritten += n; nwritten += n;
} else if (o->encoding == OBJ_ENCODING_LISTPACK) {
size_t l = lpBytes((unsigned char *)o->ptr);
if ((n = rdbSaveRawString(rdb, o->ptr, l)) == -1) return -1;
nwritten += n;
} else { } else {
serverPanic("Unknown set encoding"); serverPanic("Unknown set encoding");
} }
...@@ -1690,19 +1696,21 @@ static int _listZiplistEntryConvertAndValidate(unsigned char *p, unsigned int he ...@@ -1690,19 +1696,21 @@ static int _listZiplistEntryConvertAndValidate(unsigned char *p, unsigned int he
} }
/* callback for to check the listpack doesn't have duplicate records */ /* callback for to check the listpack doesn't have duplicate records */
static int _lpPairsEntryValidation(unsigned char *p, unsigned int head_count, void *userdata) { static int _lpEntryValidation(unsigned char *p, unsigned int head_count, void *userdata) {
struct { struct {
int pairs;
long count; long count;
dict *fields; dict *fields;
} *data = userdata; } *data = userdata;
if (data->fields == NULL) { if (data->fields == NULL) {
data->fields = dictCreate(&hashDictType); data->fields = dictCreate(&hashDictType);
dictExpand(data->fields, head_count/2); dictExpand(data->fields, data->pairs ? head_count/2 : head_count);
} }
/* Even records are field names, add to dict and check that's not a dup */ /* If we're checking pairs, then even records are field names. Otherwise
if (((data->count) & 1) == 0) { * we're checking all elements. Add to dict and check that's not a dup */
if (!data->pairs || ((data->count) & 1) == 0) {
unsigned char *str; unsigned char *str;
int64_t slen; int64_t slen;
unsigned char buf[LP_INTBUF_SIZE]; unsigned char buf[LP_INTBUF_SIZE];
...@@ -1722,21 +1730,24 @@ static int _lpPairsEntryValidation(unsigned char *p, unsigned int head_count, vo ...@@ -1722,21 +1730,24 @@ static int _lpPairsEntryValidation(unsigned char *p, unsigned int head_count, vo
/* Validate the integrity of the listpack structure. /* Validate the integrity of the listpack structure.
* when `deep` is 0, only the integrity of the header is validated. * when `deep` is 0, only the integrity of the header is validated.
* when `deep` is 1, we scan all the entries one by one. */ * when `deep` is 1, we scan all the entries one by one.
int lpPairsValidateIntegrityAndDups(unsigned char *lp, size_t size, int deep) { * when `pairs` is 0, all elements need to be unique (it's a set)
* when `pairs` is 1, odd elements need to be unique (it's a key-value map) */
int lpValidateIntegrityAndDups(unsigned char *lp, size_t size, int deep, int pairs) {
if (!deep) if (!deep)
return lpValidateIntegrity(lp, size, 0, NULL, NULL); return lpValidateIntegrity(lp, size, 0, NULL, NULL);
/* Keep track of the field names to locate duplicate ones */ /* Keep track of the field names to locate duplicate ones */
struct { struct {
int pairs;
long count; long count;
dict *fields; /* Initialisation at the first callback. */ dict *fields; /* Initialisation at the first callback. */
} data = {0, NULL}; } data = {pairs, 0, NULL};
int ret = lpValidateIntegrity(lp, size, 1, _lpPairsEntryValidation, &data); int ret = lpValidateIntegrity(lp, size, 1, _lpEntryValidation, &data);
/* make sure we have an even number of records. */ /* make sure we have an even number of records. */
if (data.count & 1) if (pairs && data.count & 1)
ret = 0; ret = 0;
if (data.fields) dictRelease(data.fields); if (data.fields) dictRelease(data.fields);
...@@ -1813,6 +1824,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1813,6 +1824,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
} }
/* Load every single element of the set */ /* Load every single element of the set */
size_t maxelelen = 0, sumelelen = 0;
for (i = 0; i < len; i++) { for (i = 0; i < len; i++) {
long long llval; long long llval;
sds sdsele; sds sdsele;
...@@ -1821,6 +1833,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1821,6 +1833,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
decrRefCount(o); decrRefCount(o);
return NULL; return NULL;
} }
size_t elelen = sdslen(sdsele);
sumelelen += elelen;
if (elelen > maxelelen) maxelelen = elelen;
if (o->encoding == OBJ_ENCODING_INTSET) { if (o->encoding == OBJ_ENCODING_INTSET) {
/* Fetch integer value from element. */ /* Fetch integer value from element. */
...@@ -1833,6 +1848,14 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1833,6 +1848,14 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
sdsfree(sdsele); sdsfree(sdsele);
return NULL; return NULL;
} }
} else if (setTypeSize(o) < server.set_max_listpack_entries &&
maxelelen <= server.set_max_listpack_value &&
lpSafeToAdd(NULL, sumelelen))
{
/* We checked if it's safe to add one large element instead
* of many small ones. It's OK since lpSafeToAdd doesn't
* care about individual elements, only the total size. */
setTypeConvert(o, OBJ_ENCODING_LISTPACK);
} else { } else {
setTypeConvert(o,OBJ_ENCODING_HT); setTypeConvert(o,OBJ_ENCODING_HT);
if (dictTryExpand(o->ptr,len) != DICT_OK) { if (dictTryExpand(o->ptr,len) != DICT_OK) {
...@@ -1844,6 +1867,33 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1844,6 +1867,33 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
} }
} }
/* This will also be called when the set was just converted
* to a listpack encoded set. */
if (o->encoding == OBJ_ENCODING_LISTPACK) {
if (setTypeSize(o) < server.set_max_listpack_entries &&
elelen <= server.set_max_listpack_value &&
lpSafeToAdd(o->ptr, elelen))
{
unsigned char *p = lpFirst(o->ptr);
if (p && lpFind(o->ptr, p, (unsigned char*)sdsele, elelen, 0)) {
rdbReportCorruptRDB("Duplicate set members detected");
decrRefCount(o);
sdsfree(sdsele);
return NULL;
}
o->ptr = lpAppend(o->ptr, (unsigned char *)sdsele, elelen);
} else {
setTypeConvert(o, OBJ_ENCODING_HT);
if (dictTryExpand(o->ptr, len) != DICT_OK) {
rdbReportCorruptRDB("OOM in dictTryExpand %llu",
(unsigned long long)len);
sdsfree(sdsele);
decrRefCount(o);
return NULL;
}
}
}
/* This will also be called when the set was just converted /* This will also be called when the set was just converted
* to a regular hash table encoded set. */ * to a regular hash table encoded set. */
if (o->encoding == OBJ_ENCODING_HT) { if (o->encoding == OBJ_ENCODING_HT) {
...@@ -2126,6 +2176,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -2126,6 +2176,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
} else if (rdbtype == RDB_TYPE_HASH_ZIPMAP || } else if (rdbtype == RDB_TYPE_HASH_ZIPMAP ||
rdbtype == RDB_TYPE_LIST_ZIPLIST || rdbtype == RDB_TYPE_LIST_ZIPLIST ||
rdbtype == RDB_TYPE_SET_INTSET || rdbtype == RDB_TYPE_SET_INTSET ||
rdbtype == RDB_TYPE_SET_LISTPACK ||
rdbtype == RDB_TYPE_ZSET_ZIPLIST || rdbtype == RDB_TYPE_ZSET_ZIPLIST ||
rdbtype == RDB_TYPE_ZSET_LISTPACK || rdbtype == RDB_TYPE_ZSET_LISTPACK ||
rdbtype == RDB_TYPE_HASH_ZIPLIST || rdbtype == RDB_TYPE_HASH_ZIPLIST ||
...@@ -2243,6 +2294,20 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -2243,6 +2294,20 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
if (intsetLen(o->ptr) > server.set_max_intset_entries) if (intsetLen(o->ptr) > server.set_max_intset_entries)
setTypeConvert(o,OBJ_ENCODING_HT); setTypeConvert(o,OBJ_ENCODING_HT);
break; break;
case RDB_TYPE_SET_LISTPACK:
if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
if (!lpValidateIntegrityAndDups(encoded, encoded_len, deep_integrity_validation, 0)) {
rdbReportCorruptRDB("Set listpack integrity check failed.");
zfree(encoded);
o->ptr = NULL;
decrRefCount(o);
return NULL;
}
o->type = OBJ_SET;
o->encoding = OBJ_ENCODING_LISTPACK;
if (setTypeSize(o) > server.set_max_listpack_entries)
setTypeConvert(o, OBJ_ENCODING_HT);
break;
case RDB_TYPE_ZSET_ZIPLIST: case RDB_TYPE_ZSET_ZIPLIST:
{ {
unsigned char *lp = lpNew(encoded_len); unsigned char *lp = lpNew(encoded_len);
...@@ -2272,7 +2337,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -2272,7 +2337,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
} }
case RDB_TYPE_ZSET_LISTPACK: case RDB_TYPE_ZSET_LISTPACK:
if (deep_integrity_validation) server.stat_dump_payload_sanitizations++; if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
if (!lpPairsValidateIntegrityAndDups(encoded, encoded_len, deep_integrity_validation)) { if (!lpValidateIntegrityAndDups(encoded, encoded_len, deep_integrity_validation, 1)) {
rdbReportCorruptRDB("Zset listpack integrity check failed."); rdbReportCorruptRDB("Zset listpack integrity check failed.");
zfree(encoded); zfree(encoded);
o->ptr = NULL; o->ptr = NULL;
...@@ -2318,7 +2383,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -2318,7 +2383,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
} }
case RDB_TYPE_HASH_LISTPACK: case RDB_TYPE_HASH_LISTPACK:
if (deep_integrity_validation) server.stat_dump_payload_sanitizations++; if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
if (!lpPairsValidateIntegrityAndDups(encoded, encoded_len, deep_integrity_validation)) { if (!lpValidateIntegrityAndDups(encoded, encoded_len, deep_integrity_validation, 1)) {
rdbReportCorruptRDB("Hash listpack integrity check failed."); rdbReportCorruptRDB("Hash listpack integrity check failed.");
zfree(encoded); zfree(encoded);
o->ptr = NULL; o->ptr = NULL;
......
...@@ -38,7 +38,7 @@ ...@@ -38,7 +38,7 @@
/* The current RDB version. When the format changes in a way that is no longer /* The current RDB version. When the format changes in a way that is no longer
* backward compatible this number gets incremented. */ * backward compatible this number gets incremented. */
#define RDB_VERSION 10 #define RDB_VERSION 11
/* Defines related to the dump file format. To store 32 bits lengths for short /* Defines related to the dump file format. To store 32 bits lengths for short
* keys requires a lot of space, so we check the most significant 2 bits of * keys requires a lot of space, so we check the most significant 2 bits of
...@@ -95,10 +95,11 @@ ...@@ -95,10 +95,11 @@
#define RDB_TYPE_ZSET_LISTPACK 17 #define RDB_TYPE_ZSET_LISTPACK 17
#define RDB_TYPE_LIST_QUICKLIST_2 18 #define RDB_TYPE_LIST_QUICKLIST_2 18
#define RDB_TYPE_STREAM_LISTPACKS_2 19 #define RDB_TYPE_STREAM_LISTPACKS_2 19
#define RDB_TYPE_SET_LISTPACK 20
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
/* Test if a type is an object type. */ /* Test if a type is an object type. */
#define rdbIsObjectType(t) (((t) >= 0 && (t) <= 7) || ((t) >= 9 && (t) <= 19)) #define rdbIsObjectType(t) (((t) >= 0 && (t) <= 7) || ((t) >= 9 && (t) <= 20))
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
#define RDB_OPCODE_FUNCTION2 245 /* function library data */ #define RDB_OPCODE_FUNCTION2 245 /* function library data */
......
...@@ -97,7 +97,8 @@ char *rdb_type_string[] = { ...@@ -97,7 +97,8 @@ char *rdb_type_string[] = {
"stream", "stream",
"hash-listpack", "hash-listpack",
"zset-listpack", "zset-listpack",
"quicklist-v2" "quicklist-v2",
"set-listpack",
}; };
/* Show a few stats collected into 'rdbstate' */ /* Show a few stats collected into 'rdbstate' */
......
...@@ -1850,6 +1850,8 @@ struct redisServer { ...@@ -1850,6 +1850,8 @@ struct redisServer {
size_t hash_max_listpack_entries; size_t hash_max_listpack_entries;
size_t hash_max_listpack_value; size_t hash_max_listpack_value;
size_t set_max_intset_entries; size_t set_max_intset_entries;
size_t set_max_listpack_entries;
size_t set_max_listpack_value;
size_t zset_max_listpack_entries; size_t zset_max_listpack_entries;
size_t zset_max_listpack_value; size_t zset_max_listpack_value;
size_t hll_sparse_max_bytes; size_t hll_sparse_max_bytes;
...@@ -2331,6 +2333,7 @@ typedef struct { ...@@ -2331,6 +2333,7 @@ typedef struct {
int encoding; int encoding;
int ii; /* intset iterator */ int ii; /* intset iterator */
dictIterator *di; dictIterator *di;
unsigned char *lpi; /* listpack iterator */
} setTypeIterator; } setTypeIterator;
/* Structure to hold hash iteration abstraction. Note that iteration over /* Structure to hold hash iteration abstraction. Note that iteration over
...@@ -2655,6 +2658,7 @@ robj *createStringObjectFromLongDouble(long double value, int humanfriendly); ...@@ -2655,6 +2658,7 @@ robj *createStringObjectFromLongDouble(long double value, int humanfriendly);
robj *createQuicklistObject(void); robj *createQuicklistObject(void);
robj *createSetObject(void); robj *createSetObject(void);
robj *createIntsetObject(void); robj *createIntsetObject(void);
robj *createSetListpackObject(void);
robj *createHashObject(void); robj *createHashObject(void);
robj *createZsetObject(void); robj *createZsetObject(void);
robj *createZsetListpackObject(void); robj *createZsetListpackObject(void);
...@@ -2980,9 +2984,9 @@ int setTypeRemove(robj *subject, sds value); ...@@ -2980,9 +2984,9 @@ int setTypeRemove(robj *subject, sds value);
int setTypeIsMember(robj *subject, sds value); int setTypeIsMember(robj *subject, sds value);
setTypeIterator *setTypeInitIterator(robj *subject); setTypeIterator *setTypeInitIterator(robj *subject);
void setTypeReleaseIterator(setTypeIterator *si); void setTypeReleaseIterator(setTypeIterator *si);
int setTypeNext(setTypeIterator *si, sds *sdsele, int64_t *llele); int setTypeNext(setTypeIterator *si, char **str, size_t *len, int64_t *llele);
sds setTypeNextObject(setTypeIterator *si); sds setTypeNextObject(setTypeIterator *si);
int setTypeRandomElement(robj *setobj, sds *sdsele, int64_t *llele); int setTypeRandomElement(robj *setobj, char **str, size_t *len, int64_t *llele);
unsigned long setTypeRandomElements(robj *set, unsigned long count, robj *aux_set); unsigned long setTypeRandomElements(robj *set, unsigned long count, robj *aux_set);
unsigned long setTypeSize(const robj *subject); unsigned long setTypeSize(const robj *subject);
void setTypeConvert(robj *subject, int enc); void setTypeConvert(robj *subject, int enc);
......
This diff is collapsed.
...@@ -1971,6 +1971,10 @@ typedef struct { ...@@ -1971,6 +1971,10 @@ typedef struct {
dictIterator *di; dictIterator *di;
dictEntry *de; dictEntry *de;
} ht; } ht;
struct {
unsigned char *lp;
unsigned char *p;
} lp;
} set; } set;
/* Sorted set iterators. */ /* Sorted set iterators. */
...@@ -2025,6 +2029,9 @@ void zuiInitIterator(zsetopsrc *op) { ...@@ -2025,6 +2029,9 @@ void zuiInitIterator(zsetopsrc *op) {
it->ht.dict = op->subject->ptr; it->ht.dict = op->subject->ptr;
it->ht.di = dictGetIterator(op->subject->ptr); it->ht.di = dictGetIterator(op->subject->ptr);
it->ht.de = dictNext(it->ht.di); it->ht.de = dictNext(it->ht.di);
} else if (op->encoding == OBJ_ENCODING_LISTPACK) {
it->lp.lp = op->subject->ptr;
it->lp.p = lpFirst(it->lp.lp);
} else { } else {
serverPanic("Unknown set encoding"); serverPanic("Unknown set encoding");
} }
...@@ -2061,6 +2068,8 @@ void zuiClearIterator(zsetopsrc *op) { ...@@ -2061,6 +2068,8 @@ void zuiClearIterator(zsetopsrc *op) {
UNUSED(it); /* skip */ UNUSED(it); /* skip */
} else if (op->encoding == OBJ_ENCODING_HT) { } else if (op->encoding == OBJ_ENCODING_HT) {
dictReleaseIterator(it->ht.di); dictReleaseIterator(it->ht.di);
} else if (op->encoding == OBJ_ENCODING_LISTPACK) {
UNUSED(it);
} else { } else {
serverPanic("Unknown set encoding"); serverPanic("Unknown set encoding");
} }
...@@ -2091,14 +2100,7 @@ unsigned long zuiLength(zsetopsrc *op) { ...@@ -2091,14 +2100,7 @@ unsigned long zuiLength(zsetopsrc *op) {
return 0; return 0;
if (op->type == OBJ_SET) { if (op->type == OBJ_SET) {
if (op->encoding == OBJ_ENCODING_INTSET) { return setTypeSize(op->subject);
return intsetLen(op->subject->ptr);
} else if (op->encoding == OBJ_ENCODING_HT) {
dict *ht = op->subject->ptr;
return dictSize(ht);
} else {
serverPanic("Unknown set encoding");
}
} else if (op->type == OBJ_ZSET) { } else if (op->type == OBJ_ZSET) {
if (op->encoding == OBJ_ENCODING_LISTPACK) { if (op->encoding == OBJ_ENCODING_LISTPACK) {
return zzlLength(op->subject->ptr); return zzlLength(op->subject->ptr);
...@@ -2144,6 +2146,14 @@ int zuiNext(zsetopsrc *op, zsetopval *val) { ...@@ -2144,6 +2146,14 @@ int zuiNext(zsetopsrc *op, zsetopval *val) {
/* Move to next element. */ /* Move to next element. */
it->ht.de = dictNext(it->ht.di); it->ht.de = dictNext(it->ht.di);
} else if (op->encoding == OBJ_ENCODING_LISTPACK) {
if (it->lp.p == NULL)
return 0;
val->estr = lpGetValue(it->lp.p, &val->elen, &val->ell);
val->score = 1.0;
/* Move to next element. */
it->lp.p = lpNext(it->lp.lp, it->lp.p);
} else { } else {
serverPanic("Unknown set encoding"); serverPanic("Unknown set encoding");
} }
......
...@@ -256,30 +256,27 @@ start_server {tags {"keyspace"}} { ...@@ -256,30 +256,27 @@ start_server {tags {"keyspace"}} {
assert_equal $digest [debug_digest_value mynewlist{t}] assert_equal $digest [debug_digest_value mynewlist{t}]
} }
test {COPY basic usage for intset set} { foreach type {intset listpack hashtable} {
r del set1{t} newset1{t} test {COPY basic usage for $type set} {
r sadd set1{t} 1 2 3 r del set1{t} newset1{t}
assert_encoding intset set1{t} r sadd set1{t} 1 2 3
r copy set1{t} newset1{t} if {$type ne "intset"} {
set digest [debug_digest_value set1{t}] r sadd set1{t} a
assert_equal $digest [debug_digest_value newset1{t}] }
assert_refcount 1 set1{t} if {$type eq "hashtable"} {
assert_refcount 1 newset1{t} for {set i 4} {$i < 200} {incr i} {
r del set1{t} r sadd set1{t} $i
assert_equal $digest [debug_digest_value newset1{t}] }
} }
assert_encoding $type set1{t}
test {COPY basic usage for hashtable set} { r copy set1{t} newset1{t}
r del set2{t} newset2{t} set digest [debug_digest_value set1{t}]
r sadd set2{t} 1 2 3 a assert_equal $digest [debug_digest_value newset1{t}]
assert_encoding hashtable set2{t} assert_refcount 1 set1{t}
r copy set2{t} newset2{t} assert_refcount 1 newset1{t}
set digest [debug_digest_value set2{t}] r del set1{t}
assert_equal $digest [debug_digest_value newset2{t}] assert_equal $digest [debug_digest_value newset1{t}]
assert_refcount 1 set2{t} }
assert_refcount 1 newset2{t}
r del set2{t}
assert_equal $digest [debug_digest_value newset2{t}]
} }
test {COPY basic usage for listpack sorted set} { test {COPY basic usage for listpack sorted set} {
......
...@@ -98,7 +98,7 @@ start_server {tags {"scan network"}} { ...@@ -98,7 +98,7 @@ start_server {tags {"scan network"}} {
assert_equal 1000 [llength $keys] assert_equal 1000 [llength $keys]
} }
foreach enc {intset hashtable} { foreach enc {intset listpack hashtable} {
test "SSCAN with encoding $enc" { test "SSCAN with encoding $enc" {
# Create the Set # Create the Set
r del set r del set
...@@ -107,8 +107,9 @@ start_server {tags {"scan network"}} { ...@@ -107,8 +107,9 @@ start_server {tags {"scan network"}} {
} else { } else {
set prefix "ele:" set prefix "ele:"
} }
set count [expr {$enc eq "hashtable" ? 200 : 100}]
set elements {} set elements {}
for {set j 0} {$j < 100} {incr j} { for {set j 0} {$j < $count} {incr j} {
lappend elements ${prefix}${j} lappend elements ${prefix}${j}
} }
r sadd set {*}$elements r sadd set {*}$elements
...@@ -128,7 +129,7 @@ start_server {tags {"scan network"}} { ...@@ -128,7 +129,7 @@ start_server {tags {"scan network"}} {
} }
set keys [lsort -unique $keys] set keys [lsort -unique $keys]
assert_equal 100 [llength $keys] assert_equal $count [llength $keys]
} }
} }
......
...@@ -2,6 +2,8 @@ start_server { ...@@ -2,6 +2,8 @@ start_server {
tags {"set"} tags {"set"}
overrides { overrides {
"set-max-intset-entries" 512 "set-max-intset-entries" 512
"set-max-listpack-entries" 128
"set-max-listpack-value" 32
} }
} { } {
proc create_set {key entries} { proc create_set {key entries} {
...@@ -9,12 +11,19 @@ start_server { ...@@ -9,12 +11,19 @@ start_server {
foreach entry $entries { r sadd $key $entry } foreach entry $entries { r sadd $key $entry }
} }
test {SADD, SCARD, SISMEMBER, SMISMEMBER, SMEMBERS basics - regular set} { # Values for initialing sets, per encoding.
create_set myset {foo} array set initelems {listpack {foo} hashtable {foo}}
assert_encoding hashtable myset for {set i 0} {$i < 130} {incr i} {
lappend initelems(hashtable) [format "i%03d" $i]
}
foreach type {listpack hashtable} {
test "SADD, SCARD, SISMEMBER, SMISMEMBER, SMEMBERS basics - $type" {
create_set myset $initelems($type)
assert_encoding $type myset
assert_equal 1 [r sadd myset bar] assert_equal 1 [r sadd myset bar]
assert_equal 0 [r sadd myset bar] assert_equal 0 [r sadd myset bar]
assert_equal 2 [r scard myset] assert_equal [expr [llength $initelems($type)] + 1] [r scard myset]
assert_equal 1 [r sismember myset foo] assert_equal 1 [r sismember myset foo]
assert_equal 1 [r sismember myset bar] assert_equal 1 [r sismember myset bar]
assert_equal 0 [r sismember myset bla] assert_equal 0 [r sismember myset bla]
...@@ -23,7 +32,8 @@ start_server { ...@@ -23,7 +32,8 @@ start_server {
assert_equal {1 0} [r smismember myset foo bla] assert_equal {1 0} [r smismember myset foo bla]
assert_equal {0 1} [r smismember myset bla foo] assert_equal {0 1} [r smismember myset bla foo]
assert_equal {0} [r smismember myset bla] assert_equal {0} [r smismember myset bla]
assert_equal {bar foo} [lsort [r smembers myset]] assert_equal "bar $initelems($type)" [lsort [r smembers myset]]
}
} }
test {SADD, SCARD, SISMEMBER, SMISMEMBER, SMEMBERS basics - intset} { test {SADD, SCARD, SISMEMBER, SMISMEMBER, SMEMBERS basics - intset} {
...@@ -67,15 +77,33 @@ start_server { ...@@ -67,15 +77,33 @@ start_server {
assert_error WRONGTYPE* {r sadd mylist bar} assert_error WRONGTYPE* {r sadd mylist bar}
} }
test "SADD a non-integer against an intset" { test "SADD a non-integer against a small intset" {
create_set myset {1 2 3} create_set myset {1 2 3}
assert_encoding intset myset assert_encoding intset myset
assert_equal 1 [r sadd myset a] assert_equal 1 [r sadd myset a]
assert_encoding listpack myset
}
test "SADD a non-integer against a large intset" {
create_set myset {0}
for {set i 1} {$i < 130} {incr i} {r sadd myset $i}
assert_encoding intset myset
assert_equal 1 [r sadd myset a]
assert_encoding hashtable myset assert_encoding hashtable myset
} }
test "SADD an integer larger than 64 bits" { test "SADD an integer larger than 64 bits" {
create_set myset {213244124402402314402033402} create_set myset {213244124402402314402033402}
assert_encoding listpack myset
assert_equal 1 [r sismember myset 213244124402402314402033402]
assert_equal {1} [r smismember myset 213244124402402314402033402]
}
test "SADD an integer larger than 64 bits to a large intset" {
create_set myset {0}
for {set i 1} {$i < 130} {incr i} {r sadd myset $i}
assert_encoding intset myset
r sadd myset 213244124402402314402033402
assert_encoding hashtable myset assert_encoding hashtable myset
assert_equal 1 [r sismember myset 213244124402402314402033402] assert_equal 1 [r sismember myset 213244124402402314402033402]
assert_equal {1} [r smismember myset 213244124402402314402033402] assert_equal {1} [r smismember myset 213244124402402314402033402]
...@@ -100,25 +128,32 @@ start_server { ...@@ -100,25 +128,32 @@ start_server {
r del myintset r del myintset
r del myhashset r del myhashset
r del mylargeintset r del mylargeintset
r del mysmallset
for {set i 0} {$i < 100} {incr i} { r sadd myintset $i } for {set i 0} {$i < 100} {incr i} { r sadd myintset $i }
for {set i 0} {$i < 1280} {incr i} { r sadd mylargeintset $i } for {set i 0} {$i < 1280} {incr i} { r sadd mylargeintset $i }
for {set i 0} {$i < 50} {incr i} { r sadd mysmallset [format "i%03d" $i] }
for {set i 0} {$i < 256} {incr i} { r sadd myhashset [format "i%03d" $i] } for {set i 0} {$i < 256} {incr i} { r sadd myhashset [format "i%03d" $i] }
assert_encoding intset myintset assert_encoding intset myintset
assert_encoding hashtable mylargeintset assert_encoding hashtable mylargeintset
assert_encoding listpack mysmallset
assert_encoding hashtable myhashset assert_encoding hashtable myhashset
r debug reload r debug reload
assert_encoding intset myintset assert_encoding intset myintset
assert_encoding hashtable mylargeintset assert_encoding hashtable mylargeintset
assert_encoding listpack mysmallset
assert_encoding hashtable myhashset assert_encoding hashtable myhashset
} {} {needs:debug} } {} {needs:debug}
test {SREM basics - regular set} { foreach type {listpack hashtable} {
create_set myset {foo bar ciao} test {SREM basics - $type} {
assert_encoding hashtable myset create_set myset $initelems($type)
assert_equal 0 [r srem myset qux] r sadd myset ciao
assert_equal 1 [r srem myset foo] assert_encoding $type myset
assert_equal {bar ciao} [lsort [r smembers myset]] assert_equal 0 [r srem myset qux]
assert_equal 1 [r srem myset ciao]
assert_equal $initelems($type) [lsort [r smembers myset]]
}
} }
test {SREM basics - intset} { test {SREM basics - intset} {
...@@ -177,7 +212,18 @@ start_server { ...@@ -177,7 +212,18 @@ start_server {
assert_equal 0 [r sintercard 1 non-existing-key limit 10] assert_equal 0 [r sintercard 1 non-existing-key limit 10]
} }
foreach {type} {hashtable intset} { foreach {type} {regular intset} {
# Create sets setN{t} where N = 1..5
if {$type eq "regular"} {
set smallenc listpack
set bigenc hashtable
} else {
set smallenc intset
set bigenc intset
}
# Sets 1, 2 and 4 are big; sets 3 and 5 are small.
array set encoding "1 $bigenc 2 $bigenc 3 $smallenc 4 $bigenc 5 $smallenc"
for {set i 1} {$i <= 5} {incr i} { for {set i 1} {$i <= 5} {incr i} {
r del [format "set%d{t}" $i] r del [format "set%d{t}" $i]
} }
...@@ -198,7 +244,7 @@ start_server { ...@@ -198,7 +244,7 @@ start_server {
# while the tests are running -- an extra element is added to every # while the tests are running -- an extra element is added to every
# set that determines its encoding. # set that determines its encoding.
set large 200 set large 200
if {$type eq "hashtable"} { if {$type eq "regular"} {
set large foo set large foo
} }
...@@ -206,9 +252,9 @@ start_server { ...@@ -206,9 +252,9 @@ start_server {
r sadd [format "set%d{t}" $i] $large r sadd [format "set%d{t}" $i] $large
} }
test "Generated sets must be encoded as $type" { test "Generated sets must be encoded correctly - $type" {
for {set i 1} {$i <= 5} {incr i} { for {set i 1} {$i <= 5} {incr i} {
assert_encoding $type [format "set%d{t}" $i] assert_encoding $encoding($i) [format "set%d{t}" $i]
} }
} }
...@@ -225,14 +271,14 @@ start_server { ...@@ -225,14 +271,14 @@ start_server {
test "SINTERSTORE with two sets - $type" { test "SINTERSTORE with two sets - $type" {
r sinterstore setres{t} set1{t} set2{t} r sinterstore setres{t} set1{t} set2{t}
assert_encoding $type setres{t} assert_encoding $smallenc setres{t}
assert_equal [list 195 196 197 198 199 $large] [lsort [r smembers setres{t}]] assert_equal [list 195 196 197 198 199 $large] [lsort [r smembers setres{t}]]
} }
test "SINTERSTORE with two sets, after a DEBUG RELOAD - $type" { test "SINTERSTORE with two sets, after a DEBUG RELOAD - $type" {
r debug reload r debug reload
r sinterstore setres{t} set1{t} set2{t} r sinterstore setres{t} set1{t} set2{t}
assert_encoding $type setres{t} assert_encoding $smallenc setres{t}
assert_equal [list 195 196 197 198 199 $large] [lsort [r smembers setres{t}]] assert_equal [list 195 196 197 198 199 $large] [lsort [r smembers setres{t}]]
} {} {needs:debug} } {} {needs:debug}
...@@ -243,7 +289,7 @@ start_server { ...@@ -243,7 +289,7 @@ start_server {
test "SUNIONSTORE with two sets - $type" { test "SUNIONSTORE with two sets - $type" {
r sunionstore setres{t} set1{t} set2{t} r sunionstore setres{t} set1{t} set2{t}
assert_encoding $type setres{t} assert_encoding $bigenc setres{t}
set expected [lsort -uniq "[r smembers set1{t}] [r smembers set2{t}]"] set expected [lsort -uniq "[r smembers set1{t}] [r smembers set2{t}]"]
assert_equal $expected [lsort [r smembers setres{t}]] assert_equal $expected [lsort [r smembers setres{t}]]
} }
...@@ -294,6 +340,46 @@ start_server { ...@@ -294,6 +340,46 @@ start_server {
} }
} }
test "SINTERSTORE with two listpack sets where result is intset" {
r del setres{t} set1{t} set2{t}
r sadd set1{t} a b c 1 3 6 x y z
r sadd set2{t} e f g 1 2 3 u v w
assert_encoding listpack set1{t}
assert_encoding listpack set2{t}
r sinterstore setres{t} set1{t} set2{t}
assert_equal [list 1 3] [lsort [r smembers setres{t}]]
assert_encoding intset setres{t}
}
test "SINTERSTORE with two hashtable sets where result is intset" {
r del setres{t} set1{t} set2{t}
r sadd set1{t} a b c 444 555 666
r sadd set2{t} e f g 111 222 333
set expected {}
for {set i 1} {$i < 130} {incr i} {
r sadd set1{t} $i
r sadd set2{t} $i
lappend expected $i
}
assert_encoding hashtable set1{t}
assert_encoding hashtable set2{t}
r sinterstore setres{t} set1{t} set2{t}
assert_equal [lsort $expected] [lsort [r smembers setres{t}]]
assert_encoding intset setres{t}
}
test "SUNION hashtable and listpack" {
# This adds code coverage for adding a non-sds string to a hashtable set
# which already contains the string.
r del set1{t} set2{t}
set union {abcdefghijklmnopqrstuvwxyz1234567890 a b c 1 2 3}
create_set set1{t} $union
create_set set2{t} {a b c}
assert_encoding hashtable set1{t}
assert_encoding listpack set2{t}
assert_equal [lsort $union] [lsort [r sunion set1{t} set2{t}]]
}
test "SDIFF with first set empty" { test "SDIFF with first set empty" {
r del set1{t} set2{t} set3{t} r del set1{t} set2{t} set3{t}
r sadd set2{t} 1 2 3 4 r sadd set2{t} 1 2 3 4
...@@ -428,7 +514,7 @@ start_server { ...@@ -428,7 +514,7 @@ start_server {
r sadd set2{t} 1 2 3 a r sadd set2{t} 1 2 3 a
r srem set2{t} a r srem set2{t} a
assert_encoding intset set1{t} assert_encoding intset set1{t}
assert_encoding hashtable set2{t} assert_encoding listpack set2{t}
lsort [r sinter set1{t} set2{t}] lsort [r sinter set1{t} set2{t}]
} {1 2 3} } {1 2 3}
...@@ -549,7 +635,7 @@ start_server { ...@@ -549,7 +635,7 @@ start_server {
assert_equal 0 [r exists setres{t}] assert_equal 0 [r exists setres{t}]
} }
foreach {type contents} {hashtable {a b c} intset {1 2 3}} { foreach {type contents} {listpack {a b c} intset {1 2 3}} {
test "SPOP basics - $type" { test "SPOP basics - $type" {
create_set myset $contents create_set myset $contents
assert_encoding $type myset assert_encoding $type myset
...@@ -575,11 +661,20 @@ start_server { ...@@ -575,11 +661,20 @@ start_server {
} }
} }
test "SPOP integer from listpack set" {
create_set myset {a 1 2 3 4 5 6 7}
assert_encoding listpack myset
set a [r spop myset]
set b [r spop myset]
assert {[string is digit $a] || [string is digit $b]}
}
foreach {type contents} { foreach {type contents} {
hashtable {a b c d e f g h i j k l m n o p q r s t u v w x y z} listpack {a b c d e f g h i j k l m n o p q r s t u v w x y z}
intset {1 10 11 12 13 14 15 16 17 18 19 2 20 21 22 23 24 25 26 3 4 5 6 7 8 9} intset {1 10 11 12 13 14 15 16 17 18 19 2 20 21 22 23 24 25 26 3 4 5 6 7 8 9}
hashtable {ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 b c d e f g h i j k l m n o p q r s t u v w x y z}
} { } {
test "SPOP with <count>" { test "SPOP with <count> - $type" {
create_set myset $contents create_set myset $contents
assert_encoding $type myset assert_encoding $type myset
assert_equal $contents [lsort [concat [r spop myset 11] [r spop myset 9] [r spop myset 0] [r spop myset 4] [r spop myset 1] [r spop myset 0] [r spop myset 1] [r spop myset 0]]] assert_equal $contents [lsort [concat [r spop myset 11] [r spop myset 9] [r spop myset 0] [r spop myset 4] [r spop myset 1] [r spop myset 0] [r spop myset 1] [r spop myset 0]]]
...@@ -610,16 +705,20 @@ start_server { ...@@ -610,16 +705,20 @@ start_server {
r spop nonexisting_key 100 r spop nonexisting_key 100
} {} } {}
test "SPOP new implementation: code path #1" { foreach {type content} {
set content {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20} intset {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20}
listpack {a 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20}
} {
test "SPOP new implementation: code path #1 $type" {
create_set myset $content create_set myset $content
assert_encoding $type myset
set res [r spop myset 30] set res [r spop myset 30]
assert {[lsort $content] eq [lsort $res]} assert {[lsort $content] eq [lsort $res]}
} }
test "SPOP new implementation: code path #2" { test "SPOP new implementation: code path #2 $type" {
set content {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20}
create_set myset $content create_set myset $content
assert_encoding $type myset
set res [r spop myset 2] set res [r spop myset 2]
assert {[llength $res] == 2} assert {[llength $res] == 2}
assert {[r scard myset] == 18} assert {[r scard myset] == 18}
...@@ -627,15 +726,16 @@ start_server { ...@@ -627,15 +726,16 @@ start_server {
assert {[lsort $union] eq [lsort $content]} assert {[lsort $union] eq [lsort $content]}
} }
test "SPOP new implementation: code path #3" { test "SPOP new implementation: code path #3 $type" {
set content {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20}
create_set myset $content create_set myset $content
assert_encoding $type myset
set res [r spop myset 18] set res [r spop myset 18]
assert {[llength $res] == 18} assert {[llength $res] == 18}
assert {[r scard myset] == 2} assert {[r scard myset] == 2}
set union [concat [r smembers myset] $res] set union [concat [r smembers myset] $res]
assert {[lsort $union] eq [lsort $content]} assert {[lsort $union] eq [lsort $content]}
} }
}
test "SRANDMEMBER count of 0 is handled correctly" { test "SRANDMEMBER count of 0 is handled correctly" {
r srandmember myset 0 r srandmember myset 0
...@@ -659,7 +759,7 @@ start_server { ...@@ -659,7 +759,7 @@ start_server {
r readraw 0 r readraw 0
foreach {type contents} { foreach {type contents} {
hashtable { listpack {
1 5 10 50 125 50000 33959417 4775547 65434162 1 5 10 50 125 50000 33959417 4775547 65434162
12098459 427716 483706 2726473884 72615637475 12098459 427716 483706 2726473884 72615637475
MARY PATRICIA LINDA BARBARA ELIZABETH JENNIFER MARIA MARY PATRICIA LINDA BARBARA ELIZABETH JENNIFER MARIA
...@@ -674,9 +774,20 @@ start_server { ...@@ -674,9 +774,20 @@ start_server {
30 31 32 33 34 35 36 37 38 39 30 31 32 33 34 35 36 37 38 39
40 41 42 43 44 45 46 47 48 49 40 41 42 43 44 45 46 47 48 49
} }
hashtable {
ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
1 5 10 50 125 50000 33959417 4775547 65434162
12098459 427716 483706 2726473884 72615637475
MARY PATRICIA LINDA BARBARA ELIZABETH JENNIFER MARIA
SUSAN MARGARET DOROTHY LISA NANCY KAREN BETTY HELEN
SANDRA DONNA CAROL RUTH SHARON MICHELLE LAURA SARAH
KIMBERLY DEBORAH JESSICA SHIRLEY CYNTHIA ANGELA MELISSA
BRENDA AMY ANNA REBECCA VIRGINIA
}
} { } {
test "SRANDMEMBER with <count> - $type" { test "SRANDMEMBER with <count> - $type" {
create_set myset $contents create_set myset $contents
assert_encoding $type myset
unset -nocomplain myset unset -nocomplain myset
array set myset {} array set myset {}
foreach ele [r smembers myset] { foreach ele [r smembers myset] {
...@@ -767,16 +878,22 @@ start_server { ...@@ -767,16 +878,22 @@ start_server {
} }
foreach {type contents} { foreach {type contents} {
hashtable { listpack {
1 5 10 50 125 1 5 10 50 125
MARY PATRICIA LINDA BARBARA ELIZABETH MARY PATRICIA LINDA BARBARA ELIZABETH
} }
intset { intset {
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9
} }
hashtable {
ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
1 5 10 50 125
MARY PATRICIA LINDA BARBARA
}
} { } {
test "SRANDMEMBER histogram distribution - $type" { test "SRANDMEMBER histogram distribution - $type" {
create_set myset $contents create_set myset $contents
assert_encoding $type myset
unset -nocomplain myset unset -nocomplain myset
array set myset {} array set myset {}
foreach ele [r smembers myset] { foreach ele [r smembers myset] {
...@@ -809,7 +926,7 @@ start_server { ...@@ -809,7 +926,7 @@ start_server {
r del myset3{t} myset4{t} r del myset3{t} myset4{t}
create_set myset1{t} {1 a b} create_set myset1{t} {1 a b}
create_set myset2{t} {2 3 4} create_set myset2{t} {2 3 4}
assert_encoding hashtable myset1{t} assert_encoding listpack myset1{t}
assert_encoding intset myset2{t} assert_encoding intset myset2{t}
} }
...@@ -819,7 +936,7 @@ start_server { ...@@ -819,7 +936,7 @@ start_server {
assert_equal 1 [r smove myset1{t} myset2{t} a] assert_equal 1 [r smove myset1{t} myset2{t} a]
assert_equal {1 b} [lsort [r smembers myset1{t}]] assert_equal {1 b} [lsort [r smembers myset1{t}]]
assert_equal {2 3 4 a} [lsort [r smembers myset2{t}]] assert_equal {2 3 4 a} [lsort [r smembers myset2{t}]]
assert_encoding hashtable myset2{t} assert_encoding listpack myset2{t}
# move an integer element should not convert the encoding # move an integer element should not convert the encoding
setup_move setup_move
...@@ -855,7 +972,7 @@ start_server { ...@@ -855,7 +972,7 @@ start_server {
assert_equal 1 [r smove myset1{t} myset3{t} a] assert_equal 1 [r smove myset1{t} myset3{t} a]
assert_equal {1 b} [lsort [r smembers myset1{t}]] assert_equal {1 b} [lsort [r smembers myset1{t}]]
assert_equal {a} [lsort [r smembers myset3{t}]] assert_equal {a} [lsort [r smembers myset3{t}]]
assert_encoding hashtable myset3{t} assert_encoding listpack myset3{t}
} }
test "SMOVE from intset to non existing destination set" { test "SMOVE from intset to non existing destination set" {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment