Unverified Commit 2168ccc6 authored by sundb's avatar sundb Committed by GitHub
Browse files

Add listpack encoding for list (#11303)

Improve memory efficiency of list keys

## Description of the feature
The new listpack encoding uses the old `list-max-listpack-size` config
to perform the conversion, which we can think it of as a node inside a
quicklist, but without 80 bytes overhead (internal fragmentation included)
of quicklist and quicklistNode structs.
For example, a list key with 5 items of 10 chars each, now takes 128 bytes
instead of 208 it used to take.

## Conversion rules
* Convert listpack to quicklist
  When the listpack length or size reaches the `list-max-listpack-size` limit,
  it will be converted to a quicklist.
* Convert quicklist to listpack
  When a quicklist has only one node, and its length or size is reduced to half
  of the `list-max-listpack-size` limit, it will be converted to a listpack.
  This is done to avoid frequent conversions when we add or remove at the bounding size or length.
    
## Interface changes
1. add list entry param to listTypeSetIteratorDirection
    When list encoding is listpack, `listTypeIterator->lpi` points to the next entry of current entry,
    so when changing the direction, we need to use the current node (listTypeEntry->p) to 
    update `listTypeIterator->lpi` to the next node in the reverse direction.

## Benchmark
### Listpack VS Quicklist with one node
* LPUSH - roughly 0.3% improvement
* LRANGE - roughly 13% improvement

### Both are quicklist
* LRANGE - roughly 3% improvement
* LRANGE without pipeline - roughly 3% improvement

From the benchmark, as we can see from the results
1. When list is quicklist encoding, LRANGE improves performance by <5%.
2. When list is listpack encoding, LRANGE improves performance by ~13%,
   the main enhancement is brought by `addListListpackRangeReply()`.

## Memory usage
1M lists(key:0~key:1000000) with 5 items of 10 chars ("hellohello") each.
shows memory usage down by 35.49%, from 214MB to 138MB.

## Note
1. Add conversion callback to support doing some work before conversion
    Since the quicklist iterator decompresses the current node when it is released, we can 
    no longer decompress the quicklist after we convert the list.
parent d136bf28
......@@ -1775,42 +1775,40 @@ int rioWriteBulkObject(rio *r, robj *obj) {
int rewriteListObject(rio *r, robj *key, robj *o) {
long long count = 0, items = listTypeLength(o);
if (o->encoding == OBJ_ENCODING_QUICKLIST) {
quicklist *list = o->ptr;
quicklistIter *li = quicklistGetIterator(list, AL_START_HEAD);
quicklistEntry entry;
while (quicklistNext(li,&entry)) {
if (count == 0) {
int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
AOF_REWRITE_ITEMS_PER_CMD : items;
if (!rioWriteBulkCount(r,'*',2+cmd_items) ||
!rioWriteBulkString(r,"RPUSH",5) ||
!rioWriteBulkObject(r,key))
{
quicklistReleaseIterator(li);
return 0;
}
listTypeIterator *li = listTypeInitIterator(o,0,LIST_TAIL);
listTypeEntry entry;
while (listTypeNext(li,&entry)) {
if (count == 0) {
int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
AOF_REWRITE_ITEMS_PER_CMD : items;
if (!rioWriteBulkCount(r,'*',2+cmd_items) ||
!rioWriteBulkString(r,"RPUSH",5) ||
!rioWriteBulkObject(r,key))
{
listTypeReleaseIterator(li);
return 0;
}
}
if (entry.value) {
if (!rioWriteBulkString(r,(char*)entry.value,entry.sz)) {
quicklistReleaseIterator(li);
return 0;
}
} else {
if (!rioWriteBulkLongLong(r,entry.longval)) {
quicklistReleaseIterator(li);
return 0;
}
unsigned char *vstr;
size_t vlen;
long long lval;
vstr = listTypeGetValue(&entry,&vlen,&lval);
if (vstr) {
if (!rioWriteBulkString(r,(char*)vstr,vlen)) {
listTypeReleaseIterator(li);
return 0;
}
} else {
if (!rioWriteBulkLongLong(r,lval)) {
listTypeReleaseIterator(li);
return 0;
}
if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
items--;
}
quicklistReleaseIterator(li);
} else {
serverPanic("Unknown list encoding");
if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
items--;
}
listTypeReleaseIterator(li);
return 1;
}
......
......@@ -868,6 +868,9 @@ long defragKey(redisDb *db, dictEntry *de) {
} else if (ob->type == OBJ_LIST) {
if (ob->encoding == OBJ_ENCODING_QUICKLIST) {
defragged += defragQuicklist(db, de);
} else if (ob->encoding == OBJ_ENCODING_LISTPACK) {
if ((newzl = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newzl;
} else {
serverPanic("Unknown list encoding");
}
......
......@@ -102,7 +102,7 @@ void lazyfreeResetStats() {
* For lists the function returns the number of elements in the quicklist
* representing the list. */
size_t lazyfreeGetFreeEffort(robj *key, robj *obj, int dbid) {
if (obj->type == OBJ_LIST) {
if (obj->type == OBJ_LIST && obj->encoding == OBJ_ENCODING_QUICKLIST) {
quicklist *ql = obj->ptr;
return ql->len;
} else if (obj->type == OBJ_SET && obj->encoding == OBJ_ENCODING_HT) {
......
......@@ -1209,6 +1209,13 @@ unsigned char *lpMerge(unsigned char **first, unsigned char **second) {
return target;
}
unsigned char *lpDup(unsigned char *lp) {
size_t lpbytes = lpBytes(lp);
unsigned char *newlp = lp_malloc(lpbytes);
memcpy(newlp, lp, lpbytes);
return newlp;
}
/* Return the total number of bytes the listpack is composed of. */
size_t lpBytes(unsigned char *lp) {
return lpGetTotalBytes(lp);
......
......@@ -72,6 +72,7 @@ unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsi
unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num);
unsigned char *lpBatchDelete(unsigned char *lp, unsigned char **ps, unsigned long count);
unsigned char *lpMerge(unsigned char **first, unsigned char **second);
unsigned char *lpDup(unsigned char *lp);
unsigned long lpLength(unsigned char *lp);
unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf);
unsigned char *lpGetValue(unsigned char *p, unsigned int *slen, long long *lval);
......
......@@ -625,9 +625,7 @@ int moduleCreateEmptyKey(RedisModuleKey *key, int type) {
 
switch(type) {
case REDISMODULE_KEYTYPE_LIST:
obj = createQuicklistObject();
quicklistSetOptions(obj->ptr, server.list_max_listpack_size,
server.list_compress_depth);
obj = createListListpackObject();
break;
case REDISMODULE_KEYTYPE_ZSET:
obj = createZsetListpackObject();
......@@ -660,6 +658,14 @@ static void moduleFreeKeyIterator(RedisModuleKey *key) {
key->iter = NULL;
}
 
/* Callback for listTypeTryConversion().
* Frees list iterator and sets it to NULL. */
static void moduleFreeListIterator(void *data) {
RedisModuleKey *key = (RedisModuleKey*)data;
serverAssert(key->value->type == OBJ_LIST);
if (key->iter) moduleFreeKeyIterator(key);
}
/* This function is called in low-level API implementation functions in order
* to check if the value associated with the key remained empty after an
* operation that removed elements from an aggregate data type.
......@@ -4148,7 +4154,7 @@ int moduleListIteratorSeek(RedisModuleKey *key, long index, int mode) {
 
/* Seek the iterator to the requested index. */
unsigned char dir = key->u.list.index < index ? LIST_TAIL : LIST_HEAD;
listTypeSetIteratorDirection(key->iter, dir);
listTypeSetIteratorDirection(key->iter, &key->u.list.entry, dir);
while (key->u.list.index != index) {
serverAssert(listTypeNext(key->iter, &key->u.list.entry));
key->u.list.index += dir == LIST_HEAD ? -1 : 1;
......@@ -4183,6 +4189,7 @@ int RM_ListPush(RedisModuleKey *key, int where, RedisModuleString *ele) {
if (key->value && key->value->type != OBJ_LIST) return REDISMODULE_ERR;
if (key->iter) moduleFreeKeyIterator(key);
if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_LIST);
listTypeTryConversionAppend(key->value, &ele, 0, 0, moduleFreeListIterator, key);
listTypePush(key->value, ele,
(where == REDISMODULE_LIST_HEAD) ? LIST_HEAD : LIST_TAIL);
return REDISMODULE_OK;
......@@ -4216,7 +4223,8 @@ RedisModuleString *RM_ListPop(RedisModuleKey *key, int where) {
(where == REDISMODULE_LIST_HEAD) ? LIST_HEAD : LIST_TAIL);
robj *decoded = getDecodedObject(ele);
decrRefCount(ele);
moduleDelKeyIfEmpty(key);
if (!moduleDelKeyIfEmpty(key))
listTypeTryConversion(key->value, LIST_CONV_SHRINKING, moduleFreeListIterator, key);
autoMemoryAdd(key->ctx,REDISMODULE_AM_STRING,decoded);
return decoded;
}
......@@ -4270,6 +4278,11 @@ int RM_ListSet(RedisModuleKey *key, long index, RedisModuleString *value) {
errno = EINVAL;
return REDISMODULE_ERR;
}
if (!key->value || key->value->type != OBJ_LIST) {
errno = ENOTSUP;
return REDISMODULE_ERR;
}
listTypeTryConversionAppend(key->value, &value, 0, 0, moduleFreeListIterator, key);
if (moduleListIteratorSeek(key, index, REDISMODULE_WRITE)) {
listTypeReplace(&key->u.list.entry, value);
/* A note in quicklist.c forbids use of iterator after insert, so
......@@ -4315,6 +4328,7 @@ int RM_ListInsert(RedisModuleKey *key, long index, RedisModuleString *value) {
/* Insert before the first element => push head. */
return RM_ListPush(key, REDISMODULE_LIST_HEAD, value);
}
listTypeTryConversionAppend(key->value, &value, 0, 0, moduleFreeListIterator, key);
if (moduleListIteratorSeek(key, index, REDISMODULE_WRITE)) {
int where = index < 0 ? LIST_TAIL : LIST_HEAD;
listTypeInsert(&key->u.list.entry, value, where);
......@@ -4341,6 +4355,8 @@ int RM_ListDelete(RedisModuleKey *key, long index) {
if (moduleListIteratorSeek(key, index, REDISMODULE_WRITE)) {
listTypeDelete(key->iter, &key->u.list.entry);
if (moduleDelKeyIfEmpty(key)) return REDISMODULE_OK;
listTypeTryConversion(key->value, LIST_CONV_SHRINKING, moduleFreeListIterator, key);
if (!key->iter) return REDISMODULE_OK; /* Return ASAP if iterator has been freed */
if (listTypeNext(key->iter, &key->u.list.entry)) {
/* After delete entry at position 'index', we need to update
* 'key->u.list.index' according to the following cases:
......
......@@ -233,6 +233,13 @@ robj *createQuicklistObject(void) {
return o;
}
robj *createListListpackObject(void) {
unsigned char *lp = lpNew(0);
robj *o = createObject(OBJ_LIST,lp);
o->encoding = OBJ_ENCODING_LISTPACK;
return o;
}
robj *createSetObject(void) {
dict *d = dictCreate(&setDictType);
robj *o = createObject(OBJ_SET,d);
......@@ -302,6 +309,8 @@ void freeStringObject(robj *o) {
void freeListObject(robj *o) {
if (o->encoding == OBJ_ENCODING_QUICKLIST) {
quicklistRelease(o->ptr);
} else if (o->encoding == OBJ_ENCODING_LISTPACK) {
lpFree(o->ptr);
} else {
serverPanic("Unknown list encoding type");
}
......@@ -423,6 +432,8 @@ void dismissListObject(robj *o, size_t size_hint) {
node = node->next;
}
}
} else if (o->encoding == OBJ_ENCODING_LISTPACK) {
dismissMemory(o->ptr, lpBytes((unsigned char*)o->ptr));
} else {
serverPanic("Unknown list encoding type");
}
......@@ -1005,6 +1016,8 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) {
samples++;
} while ((node = node->next) && samples < sample_size);
asize += (double)elesize/samples*ql->len;
} else if (o->encoding == OBJ_ENCODING_LISTPACK) {
asize = sizeof(*o)+zmalloc_size(o->ptr);
} else {
serverPanic("Unknown list encoding");
}
......
......@@ -30,6 +30,7 @@
#include <stdio.h>
#include <string.h> /* for memcpy */
#include <limits.h>
#include "quicklist.h"
#include "zmalloc.h"
#include "config.h"
......@@ -447,25 +448,45 @@ REDIS_STATIC void _quicklistInsertNodeAfter(quicklist *quicklist,
__quicklistInsertNode(quicklist, old_node, new_node, 1);
}
REDIS_STATIC int
_quicklistNodeSizeMeetsOptimizationRequirement(const size_t sz,
const int fill) {
if (fill >= 0)
return 0;
#define sizeMeetsSafetyLimit(sz) ((sz) <= SIZE_SAFETY_LIMIT)
size_t offset = (-fill) - 1;
if (offset < (sizeof(optimization_level) / sizeof(*optimization_level))) {
if (sz <= optimization_level[offset]) {
return 1;
} else {
return 0;
}
/* Calculate the size limit or length limit of the quicklist node
* based on 'fill', and is also used to limit list listpack. */
void quicklistNodeLimit(int fill, size_t *size, unsigned int *count) {
*size = SIZE_MAX;
*count = UINT_MAX;
if (fill >= 0) {
/* Ensure that one node have at least one entry */
*count = (fill == 0) ? 1 : fill;
} else {
return 0;
size_t offset = (-fill) - 1;
size_t max_level = sizeof(optimization_level) / sizeof(*optimization_level);
if (offset >= max_level) offset = max_level - 1;
*size = optimization_level[offset];
}
}
#define sizeMeetsSafetyLimit(sz) ((sz) <= SIZE_SAFETY_LIMIT)
/* Check if the limit of the quicklist node has been reached to determine if
* insertions, merges or other operations that would increase the size of
* the node can be performed.
* Return 1 if exceeds the limit, otherwise 0. */
int quicklistNodeExceedsLimit(int fill, size_t new_sz, unsigned int new_count) {
size_t sz_limit;
unsigned int count_limit;
quicklistNodeLimit(fill, &sz_limit, &count_limit);
if (likely(sz_limit != SIZE_MAX)) {
return new_sz > sz_limit;
} else if (count_limit != UINT_MAX) {
/* when we reach here we know that the limit is a size limit (which is
* safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
if (!sizeMeetsSafetyLimit(new_sz)) return 1;
return new_count > count_limit;
}
redis_unreachable();
}
REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node,
const int fill, const size_t sz) {
......@@ -481,16 +502,9 @@ REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node,
* Note: No need to check for overflow below since both `node->sz` and
* `sz` are to be less than 1GB after the plain/large element check above. */
size_t new_sz = node->sz + sz + SIZE_ESTIMATE_OVERHEAD;
if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill)))
return 1;
/* when we return 1 above we know that the limit is a size limit (which is
* safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
else if (!sizeMeetsSafetyLimit(new_sz))
return 0;
else if ((int)node->count < fill)
return 1;
else
if (unlikely(quicklistNodeExceedsLimit(fill, new_sz, node->count + 1)))
return 0;
return 1;
}
REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,
......@@ -505,16 +519,9 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,
/* approximate merged listpack size (- 7 to remove one listpack
* header/trailer, see LP_HDR_SIZE and LP_EOF) */
unsigned int merge_sz = a->sz + b->sz - 7;
if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill)))
return 1;
/* when we return 1 above we know that the limit is a size limit (which is
* safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
else if (!sizeMeetsSafetyLimit(merge_sz))
return 0;
else if ((int)(a->count + b->count) <= fill)
return 1;
else
if (unlikely(quicklistNodeExceedsLimit(fill, merge_sz, a->count + b->count)))
return 0;
return 1;
}
#define quicklistNodeUpdateSz(node) \
......
......@@ -192,6 +192,8 @@ int quicklistPop(quicklist *quicklist, int where, unsigned char **data,
unsigned long quicklistCount(const quicklist *ql);
int quicklistCompare(quicklistEntry *entry, unsigned char *p2, const size_t p2_len);
size_t quicklistGetLzf(const quicklistNode *node, void **data);
void quicklistNodeLimit(int fill, size_t *size, unsigned int *count);
int quicklistNodeExceedsLimit(int fill, size_t new_sz, unsigned int new_count);
void quicklistRepr(unsigned char *ql, int full);
/* bookmarks */
......
......@@ -656,7 +656,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) {
case OBJ_STRING:
return rdbSaveType(rdb,RDB_TYPE_STRING);
case OBJ_LIST:
if (o->encoding == OBJ_ENCODING_QUICKLIST)
if (o->encoding == OBJ_ENCODING_QUICKLIST || o->encoding == OBJ_ENCODING_LISTPACK)
return rdbSaveType(rdb, RDB_TYPE_LIST_QUICKLIST_2);
else
serverPanic("Unknown list encoding");
......@@ -828,6 +828,16 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) {
}
node = node->next;
}
} else if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *lp = o->ptr;
/* Save list listpack as a fake quicklist that only has a single node. */
if ((n = rdbSaveLen(rdb,1)) == -1) return -1;
nwritten += n;
if ((n = rdbSaveLen(rdb,QUICKLIST_NODE_CONTAINER_PACKED)) == -1) return -1;
nwritten += n;
if ((n = rdbSaveRawString(rdb,lp,lpBytes(lp))) == -1) return -1;
nwritten += n;
} else {
serverPanic("Unknown list encoding");
}
......@@ -1802,6 +1812,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
decrRefCount(dec);
decrRefCount(ele);
}
listTypeTryConversion(o,LIST_CONV_AUTO,NULL,NULL);
} else if (rdbtype == RDB_TYPE_SET) {
/* Read Set value */
if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
......@@ -2173,6 +2185,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
decrRefCount(o);
goto emptykey;
}
listTypeTryConversion(o,LIST_CONV_AUTO,NULL,NULL);
} else if (rdbtype == RDB_TYPE_HASH_ZIPMAP ||
rdbtype == RDB_TYPE_LIST_ZIPLIST ||
rdbtype == RDB_TYPE_SET_INTSET ||
......
......@@ -2318,12 +2318,15 @@ typedef struct {
robj *subject;
unsigned char encoding;
unsigned char direction; /* Iteration direction */
quicklistIter *iter;
unsigned char *lpi; /* listpack iterator */
quicklistIter *iter; /* quicklist iterator */
} listTypeIterator;
/* Structure for an entry while iterating over a list. */
typedef struct {
listTypeIterator *li;
unsigned char *lpe; /* Entry in listpack */
quicklistEntry entry; /* Entry in quicklist */
} listTypeEntry;
......@@ -2603,18 +2606,27 @@ robj *listTypePop(robj *subject, int where);
unsigned long listTypeLength(const robj *subject);
listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char direction);
void listTypeReleaseIterator(listTypeIterator *li);
void listTypeSetIteratorDirection(listTypeIterator *li, unsigned char direction);
void listTypeSetIteratorDirection(listTypeIterator *li, listTypeEntry *entry, unsigned char direction);
int listTypeNext(listTypeIterator *li, listTypeEntry *entry);
robj *listTypeGet(listTypeEntry *entry);
unsigned char *listTypeGetValue(listTypeEntry *entry, size_t *vlen, long long *lval);
void listTypeInsert(listTypeEntry *entry, robj *value, int where);
void listTypeReplace(listTypeEntry *entry, robj *value);
int listTypeEqual(listTypeEntry *entry, robj *o);
void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry);
robj *listTypeDup(robj *o);
int listTypeDelRange(robj *o, long start, long stop);
void listTypeDelRange(robj *o, long start, long stop);
void unblockClientWaitingData(client *c);
void popGenericCommand(client *c, int where);
void listElementsRemoved(client *c, robj *key, int where, robj *o, long count, int signal, int *deleted);
typedef enum {
LIST_CONV_AUTO,
LIST_CONV_GROWING,
LIST_CONV_SHRINKING,
} list_conv_type;
typedef void (*beforeConvertCB)(void *data);
void listTypeTryConversion(robj *o, list_conv_type lct, beforeConvertCB fn, void *data);
void listTypeTryConversionAppend(robj *o, robj **argv, int start, int end, beforeConvertCB fn, void *data);
/* MULTI/EXEC/WATCH... */
void unwatchAllKeys(client *c);
......@@ -2656,6 +2668,7 @@ robj *createStringObjectFromLongLong(long long value);
robj *createStringObjectFromLongLongForValue(long long value);
robj *createStringObjectFromLongDouble(long double value, int humanfriendly);
robj *createQuicklistObject(void);
robj *createListListpackObject(void);
robj *createSetObject(void);
robj *createIntsetObject(void);
robj *createSetListpackObject(void);
......
......@@ -546,6 +546,8 @@ void sortCommandGeneric(client *c, int readonly) {
}
}
} else {
/* We can't predict the size and encoding of the stored list, we
* assume it's a large list and then convert it at the end if needed. */
robj *sobj = createQuicklistObject();
/* STORE option specified, set the sorting result as a List object */
......@@ -578,6 +580,7 @@ void sortCommandGeneric(client *c, int readonly) {
}
}
if (outputlen) {
listTypeTryConversion(sobj,LIST_CONV_AUTO,NULL,NULL);
setKey(c,c->db,storekey,sobj,0);
notifyKeyspaceEvent(NOTIFY_LIST,"sortstore",storekey,
c->db->id);
......
This diff is collapsed.
......@@ -6,10 +6,11 @@ set server_path [tmpdir "server.rdb-encoding-test"]
exec cp tests/assets/encodings.rdb $server_path
exec cp tests/assets/list-quicklist.rdb $server_path
start_server [list overrides [list "dir" $server_path "dbfilename" "list-quicklist.rdb"]] {
start_server [list overrides [list "dir" $server_path "dbfilename" "list-quicklist.rdb" save ""]] {
test "test old version rdb file" {
r select 0
assert_equal [r get x] 7
assert_encoding listpack list
r lpop list
} {7}
}
......
......@@ -78,10 +78,16 @@ start_server {tags {"aofrw external:skip"} overrides {aof-use-rdb-preamble no}}
}
foreach d {string int} {
foreach e {quicklist} {
foreach e {listpack quicklist} {
test "AOF rewrite of list with $e encoding, $d data" {
r flushall
set len 1000
if {$e eq {listpack}} {
r config set list-max-listpack-size -2
set len 10
} else {
r config set list-max-listpack-size 10
set len 1000
}
for {set j 0} {$j < $len} {incr j} {
if {$d eq {string}} {
set data [randstring 0 16 alpha]
......
......@@ -244,10 +244,15 @@ start_server {tags {"keyspace"}} {
assert {[r get mynewkey{t}] eq "foobar"}
}
test {COPY basic usage for list} {
source "tests/unit/type/list-common.tcl"
foreach {type large} [array get largevalue] {
set origin_config [config_get_set list-max-listpack-size -1]
test "COPY basic usage for list - $type" {
r del mylist{t} mynewlist{t}
r lpush mylist{t} a b c d
r lpush mylist{t} a b $large c d
assert_encoding $type mylist{t}
r copy mylist{t} mynewlist{t}
assert_encoding $type mynewlist{t}
set digest [debug_digest_value mylist{t}]
assert_equal $digest [debug_digest_value mynewlist{t}]
assert_refcount 1 mylist{t}
......@@ -255,6 +260,8 @@ start_server {tags {"keyspace"}} {
r del mylist{t}
assert_equal $digest [debug_digest_value mynewlist{t}]
}
config_set list-max-listpack-size $origin_config
}
foreach type {intset listpack hashtable} {
test {COPY basic usage for $type set} {
......
......@@ -17,6 +17,7 @@ start_server {tags {"modules"}} {
test {Module list set, get, insert, delete} {
r del k
assert_error {WRONGTYPE Operation against a key holding the wrong kind of value*} {r list.set k 1 xyz}
r rpush k x
# insert, set, get
r list.insert k 0 foo
......@@ -76,6 +77,41 @@ start_server {tags {"modules"}} {
r list.getall k
} {bar y foo}
test {Module list - encoding conversion while inserting} {
r config set list-max-listpack-size 4
r del k
r rpush k a b c d
assert_encoding listpack k
# Converts to quicklist after inserting.
r list.edit k dii foo bar
assert_encoding quicklist k
assert_equal [r list.getall k] {foo bar b c d}
# Converts to listpack after deleting three entries.
r list.edit k ddd e
assert_encoding listpack k
assert_equal [r list.getall k] {c d}
}
test {Module list - encoding conversion while replacing} {
r config set list-max-listpack-size -1
r del k
r rpush k x y z
assert_encoding listpack k
# Converts to quicklist after replacing.
set big [string repeat "x" 4096]
r list.edit k r $big
assert_encoding quicklist k
assert_equal [r list.getall k] "$big y z"
# Converts to listpack after deleting the big entry.
r list.edit k d
assert_encoding listpack k
assert_equal [r list.getall k] {y z}
}
test {Module list - list entry and index should be updated when deletion} {
set original_config [config_get_set list-max-listpack-size 1]
......
start_server {
tags {"sort"}
overrides {
"list-max-ziplist-size" 32
"list-max-ziplist-size" 16
"set-max-intset-entries" 32
}
} {
......@@ -34,10 +34,22 @@ start_server {
set _ $result
}
proc check_sort_store_encoding {key} {
set listpack_max_size [lindex [r config get list-max-ziplist-size] 1]
# When the length or size of quicklist is less than the limit,
# it will be converted to listpack.
if {[r llen $key] <= $listpack_max_size} {
assert_encoding listpack $key
} else {
assert_encoding quicklist $key
}
}
foreach {num cmd enc title} {
16 lpush quicklist "Old Ziplist"
1000 lpush quicklist "Old Linked list"
10000 lpush quicklist "Old Big Linked list"
16 lpush listpack "Listpack"
1000 lpush quicklist "Quicklist"
10000 lpush quicklist "Big Quicklist"
16 sadd intset "Intset"
1000 sadd hashtable "Hash table"
10000 sadd hashtable "Big Hash table"
......@@ -84,14 +96,14 @@ start_server {
r sort tosort BY weight_* store sort-res
assert_equal $result [r lrange sort-res 0 -1]
assert_equal 16 [r llen sort-res]
assert_encoding quicklist sort-res
check_sort_store_encoding sort-res
} {} {cluster:skip}
test "SORT BY hash field STORE" {
r sort tosort BY wobj_*->weight store sort-res
assert_equal $result [r lrange sort-res 0 -1]
assert_equal 16 [r llen sort-res]
assert_encoding quicklist sort-res
check_sort_store_encoding sort-res
} {} {cluster:skip}
test "SORT extracts STORE correctly" {
......
# We need a value larger than list-max-ziplist-value to make sure
# the list has the right encoding when it is swapped in again.
# We need a value to make sure the list has the right encoding when it is inserted.
array set largevalue {}
set largevalue(ziplist) "hello"
set largevalue(linkedlist) [string repeat "hello" 4]
set largevalue(listpack) "hello"
set largevalue(quicklist) [string repeat "x" 8192]
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment