Unverified Commit 2168ccc6 authored by sundb's avatar sundb Committed by GitHub
Browse files

Add listpack encoding for list (#11303)

Improve memory efficiency of list keys

## Description of the feature
The new listpack encoding uses the old `list-max-listpack-size` config
to perform the conversion, which we can think it of as a node inside a
quicklist, but without 80 bytes overhead (internal fragmentation included)
of quicklist and quicklistNode structs.
For example, a list key with 5 items of 10 chars each, now takes 128 bytes
instead of 208 it used to take.

## Conversion rules
* Convert listpack to quicklist
  When the listpack length or size reaches the `list-max-listpack-size` limit,
  it will be converted to a quicklist.
* Convert quicklist to listpack
  When a quicklist has only one node, and its length or size is reduced to half
  of the `list-max-listpack-size` limit, it will be converted to a listpack.
  This is done to avoid frequent conversions when we add or remove at the bounding size or length.
    
## Interface changes
1. add list entry param to listTypeSetIteratorDirection
    When list encoding is listpack, `listTypeIterator->lpi` points to the next entry of current entry,
    so when changing the direction, we need to use the current node (listTypeEntry->p) to 
    update `listTypeIterator->lpi` to the next node in the reverse direction.

## Benchmark
### Listpack VS Quicklist with one node
* LPUSH - roughly 0.3% improvement
* LRANGE - roughly 13% improvement

### Both are quicklist
* LRANGE - roughly 3% improvement
* LRANGE without pipeline - roughly 3% improvement

From the benchmark, as we can see from the results
1. When list is quicklist encoding, LRANGE improves performance by <5%.
2. When list is listpack encoding, LRANGE improves performance by ~13%,
   the main enhancement is brought by `addListListpackRangeReply()`.

## Memory usage
1M lists(key:0~key:1000000) with 5 items of 10 chars ("hellohello") each.
shows memory usage down by 35.49%, from 214MB to 138MB.

## Note
1. Add conversion callback to support doing some work before conversion
    Since the quicklist iterator decompresses the current node when it is released, we can 
    no longer decompress the quicklist after we convert the list.
parent d136bf28
...@@ -1775,42 +1775,40 @@ int rioWriteBulkObject(rio *r, robj *obj) { ...@@ -1775,42 +1775,40 @@ int rioWriteBulkObject(rio *r, robj *obj) {
int rewriteListObject(rio *r, robj *key, robj *o) { int rewriteListObject(rio *r, robj *key, robj *o) {
long long count = 0, items = listTypeLength(o); long long count = 0, items = listTypeLength(o);
if (o->encoding == OBJ_ENCODING_QUICKLIST) { listTypeIterator *li = listTypeInitIterator(o,0,LIST_TAIL);
quicklist *list = o->ptr; listTypeEntry entry;
quicklistIter *li = quicklistGetIterator(list, AL_START_HEAD); while (listTypeNext(li,&entry)) {
quicklistEntry entry; if (count == 0) {
int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
while (quicklistNext(li,&entry)) { AOF_REWRITE_ITEMS_PER_CMD : items;
if (count == 0) { if (!rioWriteBulkCount(r,'*',2+cmd_items) ||
int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? !rioWriteBulkString(r,"RPUSH",5) ||
AOF_REWRITE_ITEMS_PER_CMD : items; !rioWriteBulkObject(r,key))
if (!rioWriteBulkCount(r,'*',2+cmd_items) || {
!rioWriteBulkString(r,"RPUSH",5) || listTypeReleaseIterator(li);
!rioWriteBulkObject(r,key)) return 0;
{
quicklistReleaseIterator(li);
return 0;
}
} }
}
if (entry.value) { unsigned char *vstr;
if (!rioWriteBulkString(r,(char*)entry.value,entry.sz)) { size_t vlen;
quicklistReleaseIterator(li); long long lval;
return 0; vstr = listTypeGetValue(&entry,&vlen,&lval);
} if (vstr) {
} else { if (!rioWriteBulkString(r,(char*)vstr,vlen)) {
if (!rioWriteBulkLongLong(r,entry.longval)) { listTypeReleaseIterator(li);
quicklistReleaseIterator(li); return 0;
return 0; }
} } else {
if (!rioWriteBulkLongLong(r,lval)) {
listTypeReleaseIterator(li);
return 0;
} }
if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
items--;
} }
quicklistReleaseIterator(li); if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
} else { items--;
serverPanic("Unknown list encoding");
} }
listTypeReleaseIterator(li);
return 1; return 1;
} }
......
...@@ -868,6 +868,9 @@ long defragKey(redisDb *db, dictEntry *de) { ...@@ -868,6 +868,9 @@ long defragKey(redisDb *db, dictEntry *de) {
} else if (ob->type == OBJ_LIST) { } else if (ob->type == OBJ_LIST) {
if (ob->encoding == OBJ_ENCODING_QUICKLIST) { if (ob->encoding == OBJ_ENCODING_QUICKLIST) {
defragged += defragQuicklist(db, de); defragged += defragQuicklist(db, de);
} else if (ob->encoding == OBJ_ENCODING_LISTPACK) {
if ((newzl = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newzl;
} else { } else {
serverPanic("Unknown list encoding"); serverPanic("Unknown list encoding");
} }
......
...@@ -102,7 +102,7 @@ void lazyfreeResetStats() { ...@@ -102,7 +102,7 @@ void lazyfreeResetStats() {
* For lists the function returns the number of elements in the quicklist * For lists the function returns the number of elements in the quicklist
* representing the list. */ * representing the list. */
size_t lazyfreeGetFreeEffort(robj *key, robj *obj, int dbid) { size_t lazyfreeGetFreeEffort(robj *key, robj *obj, int dbid) {
if (obj->type == OBJ_LIST) { if (obj->type == OBJ_LIST && obj->encoding == OBJ_ENCODING_QUICKLIST) {
quicklist *ql = obj->ptr; quicklist *ql = obj->ptr;
return ql->len; return ql->len;
} else if (obj->type == OBJ_SET && obj->encoding == OBJ_ENCODING_HT) { } else if (obj->type == OBJ_SET && obj->encoding == OBJ_ENCODING_HT) {
......
...@@ -1209,6 +1209,13 @@ unsigned char *lpMerge(unsigned char **first, unsigned char **second) { ...@@ -1209,6 +1209,13 @@ unsigned char *lpMerge(unsigned char **first, unsigned char **second) {
return target; return target;
} }
unsigned char *lpDup(unsigned char *lp) {
size_t lpbytes = lpBytes(lp);
unsigned char *newlp = lp_malloc(lpbytes);
memcpy(newlp, lp, lpbytes);
return newlp;
}
/* Return the total number of bytes the listpack is composed of. */ /* Return the total number of bytes the listpack is composed of. */
size_t lpBytes(unsigned char *lp) { size_t lpBytes(unsigned char *lp) {
return lpGetTotalBytes(lp); return lpGetTotalBytes(lp);
......
...@@ -72,6 +72,7 @@ unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsi ...@@ -72,6 +72,7 @@ unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsi
unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num); unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num);
unsigned char *lpBatchDelete(unsigned char *lp, unsigned char **ps, unsigned long count); unsigned char *lpBatchDelete(unsigned char *lp, unsigned char **ps, unsigned long count);
unsigned char *lpMerge(unsigned char **first, unsigned char **second); unsigned char *lpMerge(unsigned char **first, unsigned char **second);
unsigned char *lpDup(unsigned char *lp);
unsigned long lpLength(unsigned char *lp); unsigned long lpLength(unsigned char *lp);
unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf); unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf);
unsigned char *lpGetValue(unsigned char *p, unsigned int *slen, long long *lval); unsigned char *lpGetValue(unsigned char *p, unsigned int *slen, long long *lval);
......
...@@ -625,9 +625,7 @@ int moduleCreateEmptyKey(RedisModuleKey *key, int type) { ...@@ -625,9 +625,7 @@ int moduleCreateEmptyKey(RedisModuleKey *key, int type) {
   
switch(type) { switch(type) {
case REDISMODULE_KEYTYPE_LIST: case REDISMODULE_KEYTYPE_LIST:
obj = createQuicklistObject(); obj = createListListpackObject();
quicklistSetOptions(obj->ptr, server.list_max_listpack_size,
server.list_compress_depth);
break; break;
case REDISMODULE_KEYTYPE_ZSET: case REDISMODULE_KEYTYPE_ZSET:
obj = createZsetListpackObject(); obj = createZsetListpackObject();
...@@ -660,6 +658,14 @@ static void moduleFreeKeyIterator(RedisModuleKey *key) { ...@@ -660,6 +658,14 @@ static void moduleFreeKeyIterator(RedisModuleKey *key) {
key->iter = NULL; key->iter = NULL;
} }
   
/* Callback for listTypeTryConversion().
* Frees list iterator and sets it to NULL. */
static void moduleFreeListIterator(void *data) {
RedisModuleKey *key = (RedisModuleKey*)data;
serverAssert(key->value->type == OBJ_LIST);
if (key->iter) moduleFreeKeyIterator(key);
}
/* This function is called in low-level API implementation functions in order /* This function is called in low-level API implementation functions in order
* to check if the value associated with the key remained empty after an * to check if the value associated with the key remained empty after an
* operation that removed elements from an aggregate data type. * operation that removed elements from an aggregate data type.
...@@ -4148,7 +4154,7 @@ int moduleListIteratorSeek(RedisModuleKey *key, long index, int mode) { ...@@ -4148,7 +4154,7 @@ int moduleListIteratorSeek(RedisModuleKey *key, long index, int mode) {
   
/* Seek the iterator to the requested index. */ /* Seek the iterator to the requested index. */
unsigned char dir = key->u.list.index < index ? LIST_TAIL : LIST_HEAD; unsigned char dir = key->u.list.index < index ? LIST_TAIL : LIST_HEAD;
listTypeSetIteratorDirection(key->iter, dir); listTypeSetIteratorDirection(key->iter, &key->u.list.entry, dir);
while (key->u.list.index != index) { while (key->u.list.index != index) {
serverAssert(listTypeNext(key->iter, &key->u.list.entry)); serverAssert(listTypeNext(key->iter, &key->u.list.entry));
key->u.list.index += dir == LIST_HEAD ? -1 : 1; key->u.list.index += dir == LIST_HEAD ? -1 : 1;
...@@ -4183,6 +4189,7 @@ int RM_ListPush(RedisModuleKey *key, int where, RedisModuleString *ele) { ...@@ -4183,6 +4189,7 @@ int RM_ListPush(RedisModuleKey *key, int where, RedisModuleString *ele) {
if (key->value && key->value->type != OBJ_LIST) return REDISMODULE_ERR; if (key->value && key->value->type != OBJ_LIST) return REDISMODULE_ERR;
if (key->iter) moduleFreeKeyIterator(key); if (key->iter) moduleFreeKeyIterator(key);
if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_LIST); if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_LIST);
listTypeTryConversionAppend(key->value, &ele, 0, 0, moduleFreeListIterator, key);
listTypePush(key->value, ele, listTypePush(key->value, ele,
(where == REDISMODULE_LIST_HEAD) ? LIST_HEAD : LIST_TAIL); (where == REDISMODULE_LIST_HEAD) ? LIST_HEAD : LIST_TAIL);
return REDISMODULE_OK; return REDISMODULE_OK;
...@@ -4216,7 +4223,8 @@ RedisModuleString *RM_ListPop(RedisModuleKey *key, int where) { ...@@ -4216,7 +4223,8 @@ RedisModuleString *RM_ListPop(RedisModuleKey *key, int where) {
(where == REDISMODULE_LIST_HEAD) ? LIST_HEAD : LIST_TAIL); (where == REDISMODULE_LIST_HEAD) ? LIST_HEAD : LIST_TAIL);
robj *decoded = getDecodedObject(ele); robj *decoded = getDecodedObject(ele);
decrRefCount(ele); decrRefCount(ele);
moduleDelKeyIfEmpty(key); if (!moduleDelKeyIfEmpty(key))
listTypeTryConversion(key->value, LIST_CONV_SHRINKING, moduleFreeListIterator, key);
autoMemoryAdd(key->ctx,REDISMODULE_AM_STRING,decoded); autoMemoryAdd(key->ctx,REDISMODULE_AM_STRING,decoded);
return decoded; return decoded;
} }
...@@ -4270,6 +4278,11 @@ int RM_ListSet(RedisModuleKey *key, long index, RedisModuleString *value) { ...@@ -4270,6 +4278,11 @@ int RM_ListSet(RedisModuleKey *key, long index, RedisModuleString *value) {
errno = EINVAL; errno = EINVAL;
return REDISMODULE_ERR; return REDISMODULE_ERR;
} }
if (!key->value || key->value->type != OBJ_LIST) {
errno = ENOTSUP;
return REDISMODULE_ERR;
}
listTypeTryConversionAppend(key->value, &value, 0, 0, moduleFreeListIterator, key);
if (moduleListIteratorSeek(key, index, REDISMODULE_WRITE)) { if (moduleListIteratorSeek(key, index, REDISMODULE_WRITE)) {
listTypeReplace(&key->u.list.entry, value); listTypeReplace(&key->u.list.entry, value);
/* A note in quicklist.c forbids use of iterator after insert, so /* A note in quicklist.c forbids use of iterator after insert, so
...@@ -4315,6 +4328,7 @@ int RM_ListInsert(RedisModuleKey *key, long index, RedisModuleString *value) { ...@@ -4315,6 +4328,7 @@ int RM_ListInsert(RedisModuleKey *key, long index, RedisModuleString *value) {
/* Insert before the first element => push head. */ /* Insert before the first element => push head. */
return RM_ListPush(key, REDISMODULE_LIST_HEAD, value); return RM_ListPush(key, REDISMODULE_LIST_HEAD, value);
} }
listTypeTryConversionAppend(key->value, &value, 0, 0, moduleFreeListIterator, key);
if (moduleListIteratorSeek(key, index, REDISMODULE_WRITE)) { if (moduleListIteratorSeek(key, index, REDISMODULE_WRITE)) {
int where = index < 0 ? LIST_TAIL : LIST_HEAD; int where = index < 0 ? LIST_TAIL : LIST_HEAD;
listTypeInsert(&key->u.list.entry, value, where); listTypeInsert(&key->u.list.entry, value, where);
...@@ -4341,6 +4355,8 @@ int RM_ListDelete(RedisModuleKey *key, long index) { ...@@ -4341,6 +4355,8 @@ int RM_ListDelete(RedisModuleKey *key, long index) {
if (moduleListIteratorSeek(key, index, REDISMODULE_WRITE)) { if (moduleListIteratorSeek(key, index, REDISMODULE_WRITE)) {
listTypeDelete(key->iter, &key->u.list.entry); listTypeDelete(key->iter, &key->u.list.entry);
if (moduleDelKeyIfEmpty(key)) return REDISMODULE_OK; if (moduleDelKeyIfEmpty(key)) return REDISMODULE_OK;
listTypeTryConversion(key->value, LIST_CONV_SHRINKING, moduleFreeListIterator, key);
if (!key->iter) return REDISMODULE_OK; /* Return ASAP if iterator has been freed */
if (listTypeNext(key->iter, &key->u.list.entry)) { if (listTypeNext(key->iter, &key->u.list.entry)) {
/* After delete entry at position 'index', we need to update /* After delete entry at position 'index', we need to update
* 'key->u.list.index' according to the following cases: * 'key->u.list.index' according to the following cases:
......
...@@ -233,6 +233,13 @@ robj *createQuicklistObject(void) { ...@@ -233,6 +233,13 @@ robj *createQuicklistObject(void) {
return o; return o;
} }
robj *createListListpackObject(void) {
unsigned char *lp = lpNew(0);
robj *o = createObject(OBJ_LIST,lp);
o->encoding = OBJ_ENCODING_LISTPACK;
return o;
}
robj *createSetObject(void) { robj *createSetObject(void) {
dict *d = dictCreate(&setDictType); dict *d = dictCreate(&setDictType);
robj *o = createObject(OBJ_SET,d); robj *o = createObject(OBJ_SET,d);
...@@ -302,6 +309,8 @@ void freeStringObject(robj *o) { ...@@ -302,6 +309,8 @@ void freeStringObject(robj *o) {
void freeListObject(robj *o) { void freeListObject(robj *o) {
if (o->encoding == OBJ_ENCODING_QUICKLIST) { if (o->encoding == OBJ_ENCODING_QUICKLIST) {
quicklistRelease(o->ptr); quicklistRelease(o->ptr);
} else if (o->encoding == OBJ_ENCODING_LISTPACK) {
lpFree(o->ptr);
} else { } else {
serverPanic("Unknown list encoding type"); serverPanic("Unknown list encoding type");
} }
...@@ -423,6 +432,8 @@ void dismissListObject(robj *o, size_t size_hint) { ...@@ -423,6 +432,8 @@ void dismissListObject(robj *o, size_t size_hint) {
node = node->next; node = node->next;
} }
} }
} else if (o->encoding == OBJ_ENCODING_LISTPACK) {
dismissMemory(o->ptr, lpBytes((unsigned char*)o->ptr));
} else { } else {
serverPanic("Unknown list encoding type"); serverPanic("Unknown list encoding type");
} }
...@@ -1005,6 +1016,8 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { ...@@ -1005,6 +1016,8 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) {
samples++; samples++;
} while ((node = node->next) && samples < sample_size); } while ((node = node->next) && samples < sample_size);
asize += (double)elesize/samples*ql->len; asize += (double)elesize/samples*ql->len;
} else if (o->encoding == OBJ_ENCODING_LISTPACK) {
asize = sizeof(*o)+zmalloc_size(o->ptr);
} else { } else {
serverPanic("Unknown list encoding"); serverPanic("Unknown list encoding");
} }
......
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
#include <stdio.h> #include <stdio.h>
#include <string.h> /* for memcpy */ #include <string.h> /* for memcpy */
#include <limits.h>
#include "quicklist.h" #include "quicklist.h"
#include "zmalloc.h" #include "zmalloc.h"
#include "config.h" #include "config.h"
...@@ -447,25 +448,45 @@ REDIS_STATIC void _quicklistInsertNodeAfter(quicklist *quicklist, ...@@ -447,25 +448,45 @@ REDIS_STATIC void _quicklistInsertNodeAfter(quicklist *quicklist,
__quicklistInsertNode(quicklist, old_node, new_node, 1); __quicklistInsertNode(quicklist, old_node, new_node, 1);
} }
REDIS_STATIC int #define sizeMeetsSafetyLimit(sz) ((sz) <= SIZE_SAFETY_LIMIT)
_quicklistNodeSizeMeetsOptimizationRequirement(const size_t sz,
const int fill) {
if (fill >= 0)
return 0;
size_t offset = (-fill) - 1; /* Calculate the size limit or length limit of the quicklist node
if (offset < (sizeof(optimization_level) / sizeof(*optimization_level))) { * based on 'fill', and is also used to limit list listpack. */
if (sz <= optimization_level[offset]) { void quicklistNodeLimit(int fill, size_t *size, unsigned int *count) {
return 1; *size = SIZE_MAX;
} else { *count = UINT_MAX;
return 0;
} if (fill >= 0) {
/* Ensure that one node have at least one entry */
*count = (fill == 0) ? 1 : fill;
} else { } else {
return 0; size_t offset = (-fill) - 1;
size_t max_level = sizeof(optimization_level) / sizeof(*optimization_level);
if (offset >= max_level) offset = max_level - 1;
*size = optimization_level[offset];
} }
} }
#define sizeMeetsSafetyLimit(sz) ((sz) <= SIZE_SAFETY_LIMIT) /* Check if the limit of the quicklist node has been reached to determine if
* insertions, merges or other operations that would increase the size of
* the node can be performed.
* Return 1 if exceeds the limit, otherwise 0. */
int quicklistNodeExceedsLimit(int fill, size_t new_sz, unsigned int new_count) {
size_t sz_limit;
unsigned int count_limit;
quicklistNodeLimit(fill, &sz_limit, &count_limit);
if (likely(sz_limit != SIZE_MAX)) {
return new_sz > sz_limit;
} else if (count_limit != UINT_MAX) {
/* when we reach here we know that the limit is a size limit (which is
* safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
if (!sizeMeetsSafetyLimit(new_sz)) return 1;
return new_count > count_limit;
}
redis_unreachable();
}
REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node, REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node,
const int fill, const size_t sz) { const int fill, const size_t sz) {
...@@ -481,16 +502,9 @@ REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node, ...@@ -481,16 +502,9 @@ REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node,
* Note: No need to check for overflow below since both `node->sz` and * Note: No need to check for overflow below since both `node->sz` and
* `sz` are to be less than 1GB after the plain/large element check above. */ * `sz` are to be less than 1GB after the plain/large element check above. */
size_t new_sz = node->sz + sz + SIZE_ESTIMATE_OVERHEAD; size_t new_sz = node->sz + sz + SIZE_ESTIMATE_OVERHEAD;
if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill))) if (unlikely(quicklistNodeExceedsLimit(fill, new_sz, node->count + 1)))
return 1;
/* when we return 1 above we know that the limit is a size limit (which is
* safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
else if (!sizeMeetsSafetyLimit(new_sz))
return 0;
else if ((int)node->count < fill)
return 1;
else
return 0; return 0;
return 1;
} }
REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a, REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,
...@@ -505,16 +519,9 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a, ...@@ -505,16 +519,9 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,
/* approximate merged listpack size (- 7 to remove one listpack /* approximate merged listpack size (- 7 to remove one listpack
* header/trailer, see LP_HDR_SIZE and LP_EOF) */ * header/trailer, see LP_HDR_SIZE and LP_EOF) */
unsigned int merge_sz = a->sz + b->sz - 7; unsigned int merge_sz = a->sz + b->sz - 7;
if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill))) if (unlikely(quicklistNodeExceedsLimit(fill, merge_sz, a->count + b->count)))
return 1;
/* when we return 1 above we know that the limit is a size limit (which is
* safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
else if (!sizeMeetsSafetyLimit(merge_sz))
return 0;
else if ((int)(a->count + b->count) <= fill)
return 1;
else
return 0; return 0;
return 1;
} }
#define quicklistNodeUpdateSz(node) \ #define quicklistNodeUpdateSz(node) \
......
...@@ -192,6 +192,8 @@ int quicklistPop(quicklist *quicklist, int where, unsigned char **data, ...@@ -192,6 +192,8 @@ int quicklistPop(quicklist *quicklist, int where, unsigned char **data,
unsigned long quicklistCount(const quicklist *ql); unsigned long quicklistCount(const quicklist *ql);
int quicklistCompare(quicklistEntry *entry, unsigned char *p2, const size_t p2_len); int quicklistCompare(quicklistEntry *entry, unsigned char *p2, const size_t p2_len);
size_t quicklistGetLzf(const quicklistNode *node, void **data); size_t quicklistGetLzf(const quicklistNode *node, void **data);
void quicklistNodeLimit(int fill, size_t *size, unsigned int *count);
int quicklistNodeExceedsLimit(int fill, size_t new_sz, unsigned int new_count);
void quicklistRepr(unsigned char *ql, int full); void quicklistRepr(unsigned char *ql, int full);
/* bookmarks */ /* bookmarks */
......
...@@ -656,7 +656,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) { ...@@ -656,7 +656,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) {
case OBJ_STRING: case OBJ_STRING:
return rdbSaveType(rdb,RDB_TYPE_STRING); return rdbSaveType(rdb,RDB_TYPE_STRING);
case OBJ_LIST: case OBJ_LIST:
if (o->encoding == OBJ_ENCODING_QUICKLIST) if (o->encoding == OBJ_ENCODING_QUICKLIST || o->encoding == OBJ_ENCODING_LISTPACK)
return rdbSaveType(rdb, RDB_TYPE_LIST_QUICKLIST_2); return rdbSaveType(rdb, RDB_TYPE_LIST_QUICKLIST_2);
else else
serverPanic("Unknown list encoding"); serverPanic("Unknown list encoding");
...@@ -828,6 +828,16 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { ...@@ -828,6 +828,16 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) {
} }
node = node->next; node = node->next;
} }
} else if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *lp = o->ptr;
/* Save list listpack as a fake quicklist that only has a single node. */
if ((n = rdbSaveLen(rdb,1)) == -1) return -1;
nwritten += n;
if ((n = rdbSaveLen(rdb,QUICKLIST_NODE_CONTAINER_PACKED)) == -1) return -1;
nwritten += n;
if ((n = rdbSaveRawString(rdb,lp,lpBytes(lp))) == -1) return -1;
nwritten += n;
} else { } else {
serverPanic("Unknown list encoding"); serverPanic("Unknown list encoding");
} }
...@@ -1802,6 +1812,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1802,6 +1812,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
decrRefCount(dec); decrRefCount(dec);
decrRefCount(ele); decrRefCount(ele);
} }
listTypeTryConversion(o,LIST_CONV_AUTO,NULL,NULL);
} else if (rdbtype == RDB_TYPE_SET) { } else if (rdbtype == RDB_TYPE_SET) {
/* Read Set value */ /* Read Set value */
if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
...@@ -2173,6 +2185,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -2173,6 +2185,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
decrRefCount(o); decrRefCount(o);
goto emptykey; goto emptykey;
} }
listTypeTryConversion(o,LIST_CONV_AUTO,NULL,NULL);
} else if (rdbtype == RDB_TYPE_HASH_ZIPMAP || } else if (rdbtype == RDB_TYPE_HASH_ZIPMAP ||
rdbtype == RDB_TYPE_LIST_ZIPLIST || rdbtype == RDB_TYPE_LIST_ZIPLIST ||
rdbtype == RDB_TYPE_SET_INTSET || rdbtype == RDB_TYPE_SET_INTSET ||
......
...@@ -2318,12 +2318,15 @@ typedef struct { ...@@ -2318,12 +2318,15 @@ typedef struct {
robj *subject; robj *subject;
unsigned char encoding; unsigned char encoding;
unsigned char direction; /* Iteration direction */ unsigned char direction; /* Iteration direction */
quicklistIter *iter;
unsigned char *lpi; /* listpack iterator */
quicklistIter *iter; /* quicklist iterator */
} listTypeIterator; } listTypeIterator;
/* Structure for an entry while iterating over a list. */ /* Structure for an entry while iterating over a list. */
typedef struct { typedef struct {
listTypeIterator *li; listTypeIterator *li;
unsigned char *lpe; /* Entry in listpack */
quicklistEntry entry; /* Entry in quicklist */ quicklistEntry entry; /* Entry in quicklist */
} listTypeEntry; } listTypeEntry;
...@@ -2603,18 +2606,27 @@ robj *listTypePop(robj *subject, int where); ...@@ -2603,18 +2606,27 @@ robj *listTypePop(robj *subject, int where);
unsigned long listTypeLength(const robj *subject); unsigned long listTypeLength(const robj *subject);
listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char direction); listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char direction);
void listTypeReleaseIterator(listTypeIterator *li); void listTypeReleaseIterator(listTypeIterator *li);
void listTypeSetIteratorDirection(listTypeIterator *li, unsigned char direction); void listTypeSetIteratorDirection(listTypeIterator *li, listTypeEntry *entry, unsigned char direction);
int listTypeNext(listTypeIterator *li, listTypeEntry *entry); int listTypeNext(listTypeIterator *li, listTypeEntry *entry);
robj *listTypeGet(listTypeEntry *entry); robj *listTypeGet(listTypeEntry *entry);
unsigned char *listTypeGetValue(listTypeEntry *entry, size_t *vlen, long long *lval);
void listTypeInsert(listTypeEntry *entry, robj *value, int where); void listTypeInsert(listTypeEntry *entry, robj *value, int where);
void listTypeReplace(listTypeEntry *entry, robj *value); void listTypeReplace(listTypeEntry *entry, robj *value);
int listTypeEqual(listTypeEntry *entry, robj *o); int listTypeEqual(listTypeEntry *entry, robj *o);
void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry); void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry);
robj *listTypeDup(robj *o); robj *listTypeDup(robj *o);
int listTypeDelRange(robj *o, long start, long stop); void listTypeDelRange(robj *o, long start, long stop);
void unblockClientWaitingData(client *c); void unblockClientWaitingData(client *c);
void popGenericCommand(client *c, int where); void popGenericCommand(client *c, int where);
void listElementsRemoved(client *c, robj *key, int where, robj *o, long count, int signal, int *deleted); void listElementsRemoved(client *c, robj *key, int where, robj *o, long count, int signal, int *deleted);
typedef enum {
LIST_CONV_AUTO,
LIST_CONV_GROWING,
LIST_CONV_SHRINKING,
} list_conv_type;
typedef void (*beforeConvertCB)(void *data);
void listTypeTryConversion(robj *o, list_conv_type lct, beforeConvertCB fn, void *data);
void listTypeTryConversionAppend(robj *o, robj **argv, int start, int end, beforeConvertCB fn, void *data);
/* MULTI/EXEC/WATCH... */ /* MULTI/EXEC/WATCH... */
void unwatchAllKeys(client *c); void unwatchAllKeys(client *c);
...@@ -2656,6 +2668,7 @@ robj *createStringObjectFromLongLong(long long value); ...@@ -2656,6 +2668,7 @@ robj *createStringObjectFromLongLong(long long value);
robj *createStringObjectFromLongLongForValue(long long value); robj *createStringObjectFromLongLongForValue(long long value);
robj *createStringObjectFromLongDouble(long double value, int humanfriendly); robj *createStringObjectFromLongDouble(long double value, int humanfriendly);
robj *createQuicklistObject(void); robj *createQuicklistObject(void);
robj *createListListpackObject(void);
robj *createSetObject(void); robj *createSetObject(void);
robj *createIntsetObject(void); robj *createIntsetObject(void);
robj *createSetListpackObject(void); robj *createSetListpackObject(void);
......
...@@ -546,6 +546,8 @@ void sortCommandGeneric(client *c, int readonly) { ...@@ -546,6 +546,8 @@ void sortCommandGeneric(client *c, int readonly) {
} }
} }
} else { } else {
/* We can't predict the size and encoding of the stored list, we
* assume it's a large list and then convert it at the end if needed. */
robj *sobj = createQuicklistObject(); robj *sobj = createQuicklistObject();
/* STORE option specified, set the sorting result as a List object */ /* STORE option specified, set the sorting result as a List object */
...@@ -578,6 +580,7 @@ void sortCommandGeneric(client *c, int readonly) { ...@@ -578,6 +580,7 @@ void sortCommandGeneric(client *c, int readonly) {
} }
} }
if (outputlen) { if (outputlen) {
listTypeTryConversion(sobj,LIST_CONV_AUTO,NULL,NULL);
setKey(c,c->db,storekey,sobj,0); setKey(c,c->db,storekey,sobj,0);
notifyKeyspaceEvent(NOTIFY_LIST,"sortstore",storekey, notifyKeyspaceEvent(NOTIFY_LIST,"sortstore",storekey,
c->db->id); c->db->id);
......
This diff is collapsed.
...@@ -6,10 +6,11 @@ set server_path [tmpdir "server.rdb-encoding-test"] ...@@ -6,10 +6,11 @@ set server_path [tmpdir "server.rdb-encoding-test"]
exec cp tests/assets/encodings.rdb $server_path exec cp tests/assets/encodings.rdb $server_path
exec cp tests/assets/list-quicklist.rdb $server_path exec cp tests/assets/list-quicklist.rdb $server_path
start_server [list overrides [list "dir" $server_path "dbfilename" "list-quicklist.rdb"]] { start_server [list overrides [list "dir" $server_path "dbfilename" "list-quicklist.rdb" save ""]] {
test "test old version rdb file" { test "test old version rdb file" {
r select 0 r select 0
assert_equal [r get x] 7 assert_equal [r get x] 7
assert_encoding listpack list
r lpop list r lpop list
} {7} } {7}
} }
......
...@@ -78,10 +78,16 @@ start_server {tags {"aofrw external:skip"} overrides {aof-use-rdb-preamble no}} ...@@ -78,10 +78,16 @@ start_server {tags {"aofrw external:skip"} overrides {aof-use-rdb-preamble no}}
} }
foreach d {string int} { foreach d {string int} {
foreach e {quicklist} { foreach e {listpack quicklist} {
test "AOF rewrite of list with $e encoding, $d data" { test "AOF rewrite of list with $e encoding, $d data" {
r flushall r flushall
set len 1000 if {$e eq {listpack}} {
r config set list-max-listpack-size -2
set len 10
} else {
r config set list-max-listpack-size 10
set len 1000
}
for {set j 0} {$j < $len} {incr j} { for {set j 0} {$j < $len} {incr j} {
if {$d eq {string}} { if {$d eq {string}} {
set data [randstring 0 16 alpha] set data [randstring 0 16 alpha]
......
...@@ -244,10 +244,15 @@ start_server {tags {"keyspace"}} { ...@@ -244,10 +244,15 @@ start_server {tags {"keyspace"}} {
assert {[r get mynewkey{t}] eq "foobar"} assert {[r get mynewkey{t}] eq "foobar"}
} }
test {COPY basic usage for list} { source "tests/unit/type/list-common.tcl"
foreach {type large} [array get largevalue] {
set origin_config [config_get_set list-max-listpack-size -1]
test "COPY basic usage for list - $type" {
r del mylist{t} mynewlist{t} r del mylist{t} mynewlist{t}
r lpush mylist{t} a b c d r lpush mylist{t} a b $large c d
assert_encoding $type mylist{t}
r copy mylist{t} mynewlist{t} r copy mylist{t} mynewlist{t}
assert_encoding $type mynewlist{t}
set digest [debug_digest_value mylist{t}] set digest [debug_digest_value mylist{t}]
assert_equal $digest [debug_digest_value mynewlist{t}] assert_equal $digest [debug_digest_value mynewlist{t}]
assert_refcount 1 mylist{t} assert_refcount 1 mylist{t}
...@@ -255,6 +260,8 @@ start_server {tags {"keyspace"}} { ...@@ -255,6 +260,8 @@ start_server {tags {"keyspace"}} {
r del mylist{t} r del mylist{t}
assert_equal $digest [debug_digest_value mynewlist{t}] assert_equal $digest [debug_digest_value mynewlist{t}]
} }
config_set list-max-listpack-size $origin_config
}
foreach type {intset listpack hashtable} { foreach type {intset listpack hashtable} {
test {COPY basic usage for $type set} { test {COPY basic usage for $type set} {
......
...@@ -17,6 +17,7 @@ start_server {tags {"modules"}} { ...@@ -17,6 +17,7 @@ start_server {tags {"modules"}} {
test {Module list set, get, insert, delete} { test {Module list set, get, insert, delete} {
r del k r del k
assert_error {WRONGTYPE Operation against a key holding the wrong kind of value*} {r list.set k 1 xyz}
r rpush k x r rpush k x
# insert, set, get # insert, set, get
r list.insert k 0 foo r list.insert k 0 foo
...@@ -76,6 +77,41 @@ start_server {tags {"modules"}} { ...@@ -76,6 +77,41 @@ start_server {tags {"modules"}} {
r list.getall k r list.getall k
} {bar y foo} } {bar y foo}
test {Module list - encoding conversion while inserting} {
r config set list-max-listpack-size 4
r del k
r rpush k a b c d
assert_encoding listpack k
# Converts to quicklist after inserting.
r list.edit k dii foo bar
assert_encoding quicklist k
assert_equal [r list.getall k] {foo bar b c d}
# Converts to listpack after deleting three entries.
r list.edit k ddd e
assert_encoding listpack k
assert_equal [r list.getall k] {c d}
}
test {Module list - encoding conversion while replacing} {
r config set list-max-listpack-size -1
r del k
r rpush k x y z
assert_encoding listpack k
# Converts to quicklist after replacing.
set big [string repeat "x" 4096]
r list.edit k r $big
assert_encoding quicklist k
assert_equal [r list.getall k] "$big y z"
# Converts to listpack after deleting the big entry.
r list.edit k d
assert_encoding listpack k
assert_equal [r list.getall k] {y z}
}
test {Module list - list entry and index should be updated when deletion} { test {Module list - list entry and index should be updated when deletion} {
set original_config [config_get_set list-max-listpack-size 1] set original_config [config_get_set list-max-listpack-size 1]
......
start_server { start_server {
tags {"sort"} tags {"sort"}
overrides { overrides {
"list-max-ziplist-size" 32 "list-max-ziplist-size" 16
"set-max-intset-entries" 32 "set-max-intset-entries" 32
} }
} { } {
...@@ -34,10 +34,22 @@ start_server { ...@@ -34,10 +34,22 @@ start_server {
set _ $result set _ $result
} }
proc check_sort_store_encoding {key} {
set listpack_max_size [lindex [r config get list-max-ziplist-size] 1]
# When the length or size of quicklist is less than the limit,
# it will be converted to listpack.
if {[r llen $key] <= $listpack_max_size} {
assert_encoding listpack $key
} else {
assert_encoding quicklist $key
}
}
foreach {num cmd enc title} { foreach {num cmd enc title} {
16 lpush quicklist "Old Ziplist" 16 lpush listpack "Listpack"
1000 lpush quicklist "Old Linked list" 1000 lpush quicklist "Quicklist"
10000 lpush quicklist "Old Big Linked list" 10000 lpush quicklist "Big Quicklist"
16 sadd intset "Intset" 16 sadd intset "Intset"
1000 sadd hashtable "Hash table" 1000 sadd hashtable "Hash table"
10000 sadd hashtable "Big Hash table" 10000 sadd hashtable "Big Hash table"
...@@ -84,14 +96,14 @@ start_server { ...@@ -84,14 +96,14 @@ start_server {
r sort tosort BY weight_* store sort-res r sort tosort BY weight_* store sort-res
assert_equal $result [r lrange sort-res 0 -1] assert_equal $result [r lrange sort-res 0 -1]
assert_equal 16 [r llen sort-res] assert_equal 16 [r llen sort-res]
assert_encoding quicklist sort-res check_sort_store_encoding sort-res
} {} {cluster:skip} } {} {cluster:skip}
test "SORT BY hash field STORE" { test "SORT BY hash field STORE" {
r sort tosort BY wobj_*->weight store sort-res r sort tosort BY wobj_*->weight store sort-res
assert_equal $result [r lrange sort-res 0 -1] assert_equal $result [r lrange sort-res 0 -1]
assert_equal 16 [r llen sort-res] assert_equal 16 [r llen sort-res]
assert_encoding quicklist sort-res check_sort_store_encoding sort-res
} {} {cluster:skip} } {} {cluster:skip}
test "SORT extracts STORE correctly" { test "SORT extracts STORE correctly" {
......
# We need a value larger than list-max-ziplist-value to make sure # We need a value to make sure the list has the right encoding when it is inserted.
# the list has the right encoding when it is swapped in again.
array set largevalue {} array set largevalue {}
set largevalue(ziplist) "hello" set largevalue(listpack) "hello"
set largevalue(linkedlist) [string repeat "hello" 4] set largevalue(quicklist) [string repeat "x" 8192]
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment